framework.py 171.0 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

Y
Yu Yang 已提交
17
import collections
X
Xin Pan 已提交
18
from collections import defaultdict
W
WangZhen 已提交
19
from collections import Iterable
Q
qiaolongfei 已提交
20
import contextlib
21
from .wrapped_decorator import signature_safe_contextmanager, wrap_decorator
P
peizhilin 已提交
22
import os
F
fengjiayi 已提交
23
import re
24
import traceback
25
import six
26

Y
Yu Yang 已提交
27
import numpy as np
28
import subprocess
S
sneaxiy 已提交
29
import multiprocessing
30
import sys
31
import logging
M
minqiyang 已提交
32
from .. import compat as cpt
33
from .proto import framework_pb2
34 35

from . import core
36
from . import unique_name
37 38
import paddle.version as fluid_version
import warnings
Y
Yu Yang 已提交
39

40
__all__ = [
41 42 43 44
    'Program',
    'default_startup_program',
    'default_main_program',
    'program_guard',
45
    'name_scope',
S
sneaxiy 已提交
46 47 48
    'cuda_places',
    'cpu_places',
    'cuda_pinned_places',
L
lujun 已提交
49
    'in_dygraph_mode',
C
chengduo 已提交
50
    'is_compiled_with_cuda',
51
    'Variable',
52
    'load_op_library',
53
    'require_version',
54
]
Y
Yu Yang 已提交
55

Q
qiaolongfei 已提交
56 57 58 59
EMPTY_VAR_NAME = core.kEmptyVarName()
TEMP_VAR_NAME = core.kTempVarName()
GRAD_VAR_SUFFIX = core.kGradVarSuffix()
ZERO_VAR_SUFFIX = core.kZeroVarSuffix()
W
Wu Yi 已提交
60 61
CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName()

L
lujun 已提交
62 63
_dygraph_tracer_ = None
_dygraph_current_expected_place_ = None
64 65


66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
def require_version(min_version, max_version=None):
    """
        Check if the installed version of PaddlePaddle is in [min_version, max_version],
        if the installed version is lower than ``min_version`` or higher than ``max_version``,
        an exception will be thrown, NO returns if the installed version is satisfied.

        Args:
            min_version (str): the minimum version required (like '1.4.0').
            max_version (str, optional): the max version required (like '1.6.0'), default is None,
                meaning any version equal or higher than ``min_version`` is acceptable.

        Returns:
            None.

        Raises:
            TypeError: if the type of ``min_version`` is not str.
            TypeError: if the type of ``max_version`` is not str or type(None).
            ValueError: if the value of ``min_version`` is not in version format.
            ValueError: if the value of ``max_version`` is not in version format or None.
            Exception: if the installed version is lower than ``min_version`` or higher than ``max_version``.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                # any version >= 0.1.0 is acceptable.
                fluid.require_version('0.1.0')

                # if 0.1.0 <= version <= 10.0.0, it is acceptable.
                fluid.require_version(min_version='0.1.0', max_version='10.0.0')
        """
    if not isinstance(min_version, str):
        raise TypeError(
            "The type of 'min_version' in require_version must be str, but received %s."
            % (type(min_version)))

    if not isinstance(max_version, (str, type(None))):
        raise TypeError(
            "The type of 'max_version' in require_version must be str or type(None), but received %s."
            % (type(max_version)))

    check_format = re.match(r'\d+(\.\d+){0,3}', min_version)
    if check_format is None or check_format.group() != min_version:
        raise ValueError(
            "The value of 'min_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', "
            "like '1.5.2.0', but received %s" % min_version)

    if max_version is not None:
        check_format = re.match(r'\d+(\.\d+){0,3}', max_version)
        if check_format is None or check_format.group() != max_version:
            raise ValueError(
                "The value of 'max_version' in require_version must be in format '\\d+(\\.\\d+){0,3}', "
                "like '1.5.2.0', but received %s" % max_version)

    version_installed = [
        fluid_version.major, fluid_version.minor, fluid_version.patch,
        fluid_version.rc
    ]
    zero_version = ['0', '0', '0', '0']

    def version_cmp(ver_a, ver_b):
        for i in six.moves.range(len(ver_a)):
            if int(ver_a[i]) > int(ver_b[i]):
                return 1
            elif int(ver_a[i]) < int(ver_b[i]):
                return -1
        return 0

    if version_cmp(version_installed, zero_version) == 0:
        if max_version is not None:
            warnings.warn(
                "PaddlePaddle version in [%s, %s] required, but %s installed. "
                "Maybe you are using a develop version, "
                "please make sure the version is good with your code." %
                (min_version, max_version, fluid_version.full_version))
        else:
            warnings.warn(
                "PaddlePaddle version %s or higher is required, but %s installed, "
                "Maybe you are using a develop version, "
                "please make sure the version is good with your code." %
                (min_version, fluid_version.full_version))
        return

    min_version_split = min_version.split('.')
    min_version_to_check = min_version_split + zero_version[len(
        min_version_split):]

    if max_version is not None:
        max_version_split = max_version.split('.')
        max_version_to_check = max_version_split + zero_version[len(
            max_version_split):]

        if version_cmp(version_installed,
                       max_version_to_check) > 0 or version_cmp(
                           version_installed, min_version_to_check) < 0:
            raise Exception(
                "VersionError: PaddlePaddle version in [%s, %s] required, but %s installed."
                % (min_version, max_version, fluid_version.full_version))
    else:
        if version_cmp(version_installed, min_version_to_check) < 0:
            raise Exception(
                "VersionError: PaddlePaddle version %s or higher is required, but %s installed, "
                "please upgrade your PaddlePaddle to %s or other higher version."
                % (min_version, fluid_version.full_version, min_version))


L
lujun 已提交
173
def in_dygraph_mode():
L
lujun 已提交
174
    """
Y
Youwei Song 已提交
175 176
    This function checks whether the program runs in dynamic graph mode or not.
    You can turn on dynamic graph mode with :ref:`api_fluid_dygraph_guard` api.
L
lujun 已提交
177 178

    Returns:
Y
Youwei Song 已提交
179
        bool: Whether the program is running in dynamic graph mode.
L
lujun 已提交
180 181 182 183

    Examples:
        .. code-block:: python

184
            import paddle.fluid as fluid
L
lujun 已提交
185
            if fluid.in_dygraph_mode():
Y
Youwei Song 已提交
186 187 188
                print('running in dygraph mode')
            else:
                print('not running in dygraph mode')
L
lujun 已提交
189 190

    """
L
lujun 已提交
191
    return _dygraph_tracer_ is not None
192 193


194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
def _dygraph_not_support_(func):
    def __impl__(*args, **kwargs):
        assert not in_dygraph_mode(
        ), "We don't support %s in Dygraph mode" % func.__name__
        return func(*args, **kwargs)

    return __impl__


def _dygraph_only_(func):
    def __impl__(*args, **kwargs):
        assert in_dygraph_mode(
        ), "We Only support %s in Dygraph mode, please use fluid.dygraph.guard() as context to run it in Dygraph Mode" % func.__name__
        return func(*args, **kwargs)

    return __impl__


dygraph_not_support = wrap_decorator(_dygraph_not_support_)
dygraph_only = wrap_decorator(_dygraph_only_)


L
lujun 已提交
216 217
def _dygraph_tracer():
    return _dygraph_tracer_
218

W
Wu Yi 已提交
219

M
minqiyang 已提交
220
def _current_expected_place():
L
lujun 已提交
221
    return _dygraph_current_expected_place_
M
minqiyang 已提交
222 223


L
Leo Chen 已提交
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
# TODO(zhiqiu): remove this function.
def _var_base_to_np(var_base):
    """	
    convert VarBase tp numpy	
    	
    Args:	
        var_base(VarBase) : the VarBase to convert	
    Returns (np.ndarray): the np.ndarray contain the value of VarBase	
    """

    warnings.warn(
        "paddle.fluid.framework._var_base_to_np is deprecated, please use var_base.numpy() instead of _var_base_to_np(var_base)."
    )

    return var_base.numpy()


S
sneaxiy 已提交
241
def _cpu_num():
242
    if "CPU_NUM" not in os.environ.keys():
C
chengduo 已提交
243 244 245 246 247 248 249 250
        if multiprocessing.cpu_count() > 1:
            sys.stderr.write(
                '!!! The CPU_NUM is not specified, you should set CPU_NUM in the environment variable list.\n'
                'CPU_NUM indicates that how many CPUPlace are used in the current task.\n'
                'And if this parameter are set as N (equal to the number of physical CPU core) the program may be faster.\n\n'
                'export CPU_NUM={} # for example, set CPU_NUM as number of physical CPU core which is {}.\n\n'
                '!!! The default number of CPU_NUM=1.\n'.format(
                    multiprocessing.cpu_count(), multiprocessing.cpu_count()))
C
chengduo 已提交
251
        os.environ['CPU_NUM'] = str(1)
252
    cpu_num = os.environ.get('CPU_NUM')
C
chengduo 已提交
253 254 255 256 257 258 259 260 261 262
    return int(cpu_num)


def _cuda_ids():
    gpus_env = os.getenv("FLAGS_selected_gpus")
    if gpus_env:
        device_ids = [int(s) for s in gpus_env.split(",")]
    else:
        device_ids = six.moves.range(core.get_cuda_device_count())
    return device_ids
S
sneaxiy 已提交
263 264


C
chengduo 已提交
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
def is_compiled_with_cuda():
    """
    Whether this whl package can be used to run the model on GPU.

    Returns (bool): support gpu or not.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            support_gpu = fluid.is_compiled_with_cuda()
    """
    return core.is_compiled_with_cuda()


S
sneaxiy 已提交
280
def cuda_places(device_ids=None):
L
lujun 已提交
281
    """
282 283 284 285 286
    **Note**:
        For multi-card tasks, please use `FLAGS_selected_gpus` environment variable to set the visible GPU device.
        The next version will fix the problem with `CUDA_VISIBLE_DEVICES` environment variable.

    This function creates a list of :code:`fluid.CUDAPlace` objects.
S
add doc  
sneaxiy 已提交
287 288

    If :code:`device_ids` is None, environment variable of
289
    :code:`FLAGS_selected_gpus` would be checked first. For example, if
S
add doc  
sneaxiy 已提交
290 291 292
    :code:`FLAGS_selected_gpus=0,1,2`, the returned list would
    be [fluid.CUDAPlace(0), fluid.CUDAPlace(1), fluid.CUDAPlace(2)].
    If :code:`FLAGS_selected_gpus` is not set, all visible
293
    gpu places would be returned according to the :code:`CUDA_VISIBLE_DEVICES` environment variable.
S
add doc  
sneaxiy 已提交
294 295

    If :code:`device_ids` is not None, it should be the device
296
    ids of GPUs. For example, if :code:`device_ids=[0,1,2]`,
S
add doc  
sneaxiy 已提交
297 298 299
    the returned list would be 
    [fluid.CUDAPlace(0), fluid.CUDAPlace(1), fluid.CUDAPlace(2)].
    
300 301
    Parameters:
        device_ids (list or tuple of int, optional): list of GPU device ids.
S
add doc  
sneaxiy 已提交
302 303

    Returns:
304
        list of fluid.CUDAPlace: Created GPU place list.
L
lujun 已提交
305 306 307 308

    Examples:
        .. code-block:: python

309
            import paddle.fluid as fluid
L
lujun 已提交
310 311 312
            cuda_places = fluid.cuda_places()

    """
S
sneaxiy 已提交
313 314 315
    assert core.is_compiled_with_cuda(), \
        "Not compiled with CUDA"
    if device_ids is None:
C
chengduo 已提交
316
        device_ids = _cuda_ids()
S
sneaxiy 已提交
317 318 319 320 321 322
    elif not isinstance(device_ids, (list, tuple)):
        device_ids = [device_ids]
    return [core.CUDAPlace(dev_id) for dev_id in device_ids]


def cpu_places(device_count=None):
L
lujun 已提交
323
    """
324
    This function creates a list of :code:`fluid.CPUPlace` objects, and returns the created list.
S
add doc  
sneaxiy 已提交
325 326 327
    
    If :code:`device_count` is None, the device count would
    be determined by environment variable :code:`CPU_NUM`. 
C
chengduo 已提交
328 329
    If :code:`CPU_NUM` is not set, the default value is 1,
    i.e. CPU_NUM=1.
330 331
    :code:`CPU_NUM` indicates the number of devices used in the current task.
    The running of the program can be accelerated if :code:`CPU_NUM` is the same as the number of physical cores.
S
add doc  
sneaxiy 已提交
332

333 334
    Parameters:
        device_count (int, optional): device number. Default: None.
S
add doc  
sneaxiy 已提交
335 336

    Returns:
337
        list of fluid.CPUPlace: Created list of CPU places.
L
lujun 已提交
338 339 340 341

    Examples:
        .. code-block:: python

342
            import paddle.fluid as fluid
L
lujun 已提交
343 344 345
            cpu_places = fluid.cpu_places()
    """

S
sneaxiy 已提交
346 347 348 349 350 351
    if device_count is None:
        device_count = _cpu_num()
    return [core.CPUPlace()] * device_count


def cuda_pinned_places(device_count=None):
L
lujun 已提交
352
    """
353
    This function creates a list of :code:`fluid.CUDAPinnedPlace` objects.
S
add doc  
sneaxiy 已提交
354 355 356

    If :code:`device_count` is None, the device count would
    be determined by environment variable :code:`CPU_NUM`. 
357 358 359 360
    If :code:`CPU_NUM` is not set, the default value is 1,
    i.e. CPU_NUM=1.
    :code:`CPU_NUM` indicates the number of devices used in the current task.
    The running of the program can be accelerated if :code:`CPU_NUM` is the same as the number of physical cores.
S
add doc  
sneaxiy 已提交
361

362 363
    Parameters:
        device_count (int, optional): device number. Default: None.
S
add doc  
sneaxiy 已提交
364 365

    Returns:
366
        list of fluid.CUDAPinnedPlace: Created list of CUDA pinned places.
L
lujun 已提交
367 368 369 370

    Examples:
        .. code-block:: python

371
            import paddle.fluid as fluid
L
lujun 已提交
372 373 374 375 376
            cuda_pinned_places_cpu_num = fluid.cuda_pinned_places()
            # or
            cuda_pinned_places = fluid.cuda_pinned_places(1)

    """
S
sneaxiy 已提交
377 378 379
    assert core.is_compiled_with_cuda(), \
        "Not compiled with CUDA"
    if device_count is None:
380 381
        device_count = len(_cuda_ids())
    return [core.CUDAPinnedPlace()] * device_count
S
sneaxiy 已提交
382 383


384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
class NameScope(object):
    def __init__(self, name="", parent=None):
        self._children = dict()
        self._name = name
        self._parent = parent

    def child(self, prefix):
        if prefix not in self._children:
            new_child = NameScope(prefix, self)
            self._children[prefix] = [new_child]
        else:
            new_child = NameScope(prefix + "_%d" % len(self._children[prefix]),
                                  self)
            self._children[prefix].append(new_child)
        return new_child

    def parent(self):
        return self._parent

    def name(self):
        return self._name


_name_scope = NameScope()


S
rename  
sneaxiy 已提交
410
@signature_safe_contextmanager
411 412 413 414
def name_scope(prefix=None):
    """
    Generate hierarchical name prefix for the operators.

T
Tao Luo 已提交
415 416 417
    Note: 
        This should only used for debugging and visualization purpose.
        Don't use it for serious analysis such as graph/program transformations.
418 419

    Args:
T
Tao Luo 已提交
420
        prefix(str, optional): prefix. Default is none.
421 422 423

    Examples:
        .. code-block:: python
T
Tink_Y 已提交
424

425
          import paddle.fluid as fluid
426
          with fluid.name_scope("s1"):
T
Tao Luo 已提交
427 428 429 430 431 432
             a = fluid.data(name='data', shape=[None, 1], dtype='int32')
             b = a + 1
             with fluid.name_scope("s2"):
                c = b * 1
             with fluid.name_scope("s3"):
                d = c / 1
433
          with fluid.name_scope("s1"):
T
Tao Luo 已提交
434
                f = fluid.layers.pow(d, 2.0)
435
          with fluid.name_scope("s4"):
T
Tao Luo 已提交
436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
                g = f - 1

          # Op are created in the default main program.  
          for op in fluid.default_main_program().block(0).ops:
              # elementwise_add is created in /s1/
              if op.type == 'elementwise_add':
                  assert op.desc.attr("op_namescope") == '/s1/'
              # elementwise_mul is created in '/s1/s2'
              elif op.type == 'elementwise_mul':
                  assert op.desc.attr("op_namescope") == '/s1/s2/'
              # elementwise_div is created in '/s1/s3'
              elif op.type == 'elementwise_div':
                  assert op.desc.attr("op_namescope") == '/s1/s3/'
              # elementwise_sum is created in '/s4'
              elif op.type == 'elementwise_sub':
                  assert op.desc.attr("op_namescope") == '/s4/'
              # pow is created in /s1_1/
              elif op.type == 'pow':
                  assert op.desc.attr("op_namescope") == '/s1_1/'
455 456
    """
    # TODO(panyx0718): Only [0-9a-z].
457
    # in dygraph we don't need namescope since it will cause mem leak
L
Leo Chen 已提交
458 459 460
    if in_dygraph_mode():
        yield
    else:
T
tianshuo78520a 已提交
461
        assert prefix, "namescope prefix can not be empty."
462 463 464 465
        global _name_scope
        _name_scope = _name_scope.child(prefix)
        yield
        _name_scope = _name_scope.parent()
466 467 468 469 470 471 472 473 474 475 476 477


def _full_name_scope():
    global _name_scope
    scope = _name_scope
    name = ""
    while scope:
        name = scope.name() + "/" + name
        scope = scope.parent()
    return name


W
Wu Yi 已提交
478 479 480
def generate_control_dev_var_name():
    import random
    return CONTROL_DEP_VAR_PREFIX + "@" + str(random.random())
Q
qiaolongfei 已提交
481 482 483 484


def grad_var_name(var_name):
    """
485 486
    Returns:
        str: gradient name for a certain var name
Q
qiaolongfei 已提交
487 488 489
    """
    return var_name + GRAD_VAR_SUFFIX

Y
Yu Yang 已提交
490

491
def convert_np_dtype_to_dtype_(np_dtype):
492 493
    """
    Convert the data type in numpy to the data type in Paddle
494

495
    Args:
496
        np_dtype(np.dtype): the data type in numpy.
497

498 499
    Returns:
        core.VarDesc.VarType: the data type in Paddle.
500 501

    """
502 503
    dtype = np.dtype(np_dtype)
    if dtype == np.float32:
504
        return core.VarDesc.VarType.FP32
505
    elif dtype == np.float64:
506
        return core.VarDesc.VarType.FP64
507
    elif dtype == np.float16:
508
        return core.VarDesc.VarType.FP16
509
    elif dtype == np.int32:
510
        return core.VarDesc.VarType.INT32
511
    elif dtype == np.int16:
512
        return core.VarDesc.VarType.INT16
513
    elif dtype == np.int64:
514
        return core.VarDesc.VarType.INT64
515
    elif dtype == np.bool:
516
        return core.VarDesc.VarType.BOOL
517 518
    elif dtype == np.uint16:
        return core.VarDesc.VarType.INT16
519 520
    elif dtype == np.uint8:
        return core.VarDesc.VarType.UINT8
Q
qingqing01 已提交
521 522
    elif dtype == np.int8:
        return core.VarDesc.VarType.INT8
523
    else:
M
minqiyang 已提交
524
        raise ValueError("Not supported numpy dtype %s" % dtype)
525 526 527


def dtype_is_floating(dtype):
528 529 530
    """
    Check the data type is floating or not.
    Args:
531
        dtype(np.dtype|core.VarDesc.VarType): data type.
532 533 534 535 536
            Could be numpy format or Paddle format

    Returns(bool): True if data type is a float value

    """
537
    if not isinstance(dtype, core.VarDesc.VarType):
538 539
        dtype = convert_np_dtype_to_dtype_(dtype)

540 541 542 543
    return dtype in [
        core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32,
        core.VarDesc.VarType.FP64
    ]
544 545


Y
Yang Yang(Tony) 已提交
546
def _debug_string_(proto, throw_on_error=True):
547 548 549 550 551 552 553 554 555 556 557
    """
    Get the debug string of a protobuf message. The message could be not
    initialized.
    Args:
        proto(google.protobuf.message.Message): The protobuf message
        throw_on_error(bool): True if raise an error when the protobuf message
            is not initialized.

    Returns(str): The debug string of the protobuf message

    """
Y
Yu Yang 已提交
558
    error_fields = list()
Y
Yang Yang(Tony) 已提交
559
    if not proto.IsInitialized(error_fields) and throw_on_error:
C
caoying03 已提交
560 561
        raise ValueError("{0} are not initialized.\nThe message is {1}:\n".
                         format(error_fields, proto))
Y
Yu Yang 已提交
562 563 564
    return proto.__str__()


565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
def _varbase_creator(type=core.VarDesc.VarType.LOD_TENSOR,
                     name=None,
                     shape=None,
                     dtype=None,
                     persistable=None,
                     **kwargs):
    if dtype is not None:
        if not isinstance(dtype, core.VarDesc.VarType):
            dtype = convert_np_dtype_to_dtype_(dtype)

    return core.VarBase(dtype if dtype else core.VarDesc.VarType.FP32,
                        list(shape) if shape else [], name, type
                        if type else core.VarDesc.VarType.LOD_TENSOR, True
                        if persistable else False)


class VariableMetaClass(type):
    @classmethod
    def __instancecheck__(cls, instance):
        t = type(instance)
        if in_dygraph_mode():

            return issubclass(t, core.VarBase)
        else:
            return issubclass(t, Variable)


class ParameterMetaClass(VariableMetaClass):
    @classmethod
    def __instancecheck__(cls, instance):
        t = type(instance)
        if in_dygraph_mode():
            return issubclass(t, ParamBase)
        else:
            return issubclass(t, Parameter)


def _getitem_impl_(var, item):
    """
    Slice the variable.

    Args:
        item(int/slice/tuple) : the index.

    Returns:
        Sliced variable
    """

    if not isinstance(item, tuple):
        item = [item]

    decrease_axis = []
    slice_axis = []
    slice_start = []
    slice_end = []
    slice_step = []
    use_strided_slice = False
    reverse_axis = []

    def fill_constant(shape, value, force_cpu=False, out=None):
        var.block.append_op(
            type='fill_constant',
            inputs={},
            outputs={'Out': [out]},
            attrs={
                'shape': shape,
                'dtype': out.dtype,
                'value': float(value),
                'force_cpu': force_cpu
            },
            stop_gradient=True)
        out.stop_gradient = True
        return out

    for dim, slice_item in enumerate(item):
        if isinstance(slice_item, slice):
            start = slice_item.start
            end = slice_item.stop
            step = slice_item.step

            if start is None and end is None and step is None:
                continue

            if step is None:
                step = 1

            if start is None and end is None:
                assert (step == -1)
                reverse_axis.append(dim)
                continue

            if start is None:
                start = 0

            if end is None:
                end = 10000000

            if step != 1:
                use_strided_slice = True

            slice_axis.append(dim)
            slice_start.append(start)
            slice_end.append(end)
            slice_step.append(step)
        else:
            decrease_axis.append(dim)
            slice_axis.append(dim)
            slice_start.append(slice_item)
            slice_step.append(1)
            if isinstance(slice_item, Variable):
                temp_1 = var.block.create_var(dtype='int32')
                fill_constant([1], 1, force_cpu=True, out=temp_1)
                temp_end = var.block.create_var(dtype='int32')
                var.block.append_op(
                    type='elementwise_add',
                    inputs={'X': slice_item,
                            'Y': temp_1},
                    outputs={'Out': temp_end},
                    attrs={'axis': -1})
                slice_end.append(temp_end)
            else:
                slice_end.append(slice_item + 1
                                 if slice_item != -1 else 10000000)

    def contain_var(one_list):
        for ele in one_list:
            if isinstance(ele, Variable):
                return True
        return False

    def get_new_list_tensor(old_list):
        new_list_tensor = []
        for dim in old_list:
            if isinstance(dim, Variable):
                dim.stop_gradient = True
                new_list_tensor.append(dim)
            else:
                assert (isinstance(dim, int))
                temp_out = var.block.create_var(dtype='int32')
                fill_constant([1], dim, force_cpu=True, out=temp_out)
                new_list_tensor.append(temp_out)
        return new_list_tensor

    inputs = {'Input': [var]}
    attrs = {
        'axes': slice_axis,
        'starts': [],
        'ends': [],
        'decrease_axis': decrease_axis
    }
    if (use_strided_slice == True):
        attrs['strides'] = []
    infer_flags = list(1 for i in range(len(slice_axis)))
L
Leo Chen 已提交
718

719
    # starts
L
Leo Chen 已提交
720
    if contain_var(slice_start):
721 722 723 724 725 726 727 728
        inputs['StartsTensorList'] = get_new_list_tensor(slice_start)
        for i, dim in enumerate(slice_start):
            if isinstance(dim, Variable):
                attrs['starts'].append(-1)
                infer_flags[i] = -1
            else:
                attrs['starts'].append(dim)
    else:
L
Leo Chen 已提交
729 730 731 732
        attrs['starts'] = slice_start

    # ends
    if contain_var(slice_end):
733 734 735 736 737 738 739
        inputs['EndsTensorList'] = get_new_list_tensor(slice_end)
        for i, dim in enumerate(slice_end):
            if isinstance(dim, Variable):
                attrs['ends'].append(-1)
                infer_flags[i] = -1
            else:
                attrs['ends'].append(dim)
L
Leo Chen 已提交
740 741 742
    else:
        attrs['ends'] = slice_end

743 744
    # strides
    if use_strided_slice == True:
L
Leo Chen 已提交
745
        if contain_var(slice_step):
746 747 748 749 750 751 752
            inputs['StridesTensorList'] = get_new_list_tensor(slice_step)
            for i, dim in enumerate(slice_step):
                if isinstance(dim, Variable):
                    attrs['strides'].append(-1)
                    infer_flags[i] = -1
                else:
                    attrs['strides'].append(dim)
L
Leo Chen 已提交
753 754
        else:
            attrs['strides'] = slice_step
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
    # infer_flags
    attrs['infer_flags'] = infer_flags

    out = var
    if use_strided_slice == False and len(slice_axis) > 0:
        # append slice_op here
        slice_out_var = var.block.create_var(
            name=unique_name.generate_with_ignorable_key(var.name + "_slice"),
            dtype=var.dtype)

        var.block.append_op(
            type="slice",
            inputs=inputs,
            outputs={'Out': [slice_out_var]},
            attrs=attrs)

        out = slice_out_var
    elif use_strided_slice == True and len(slice_axis) > 0:
        strided_slice_out_var = var.block.create_var(
            name=unique_name.generate_with_ignorable_key(var.name +
                                                         "_strided_slice"),
            dtype=var.dtype)
        var.block.append_op(
            type="strided_slice",
            inputs=inputs,
            outputs={'Out': [strided_slice_out_var]},
            attrs=attrs)

        out = strided_slice_out_var

    if len(reverse_axis) > 0:
        reverse_out_var = var.block.create_var(
            name=unique_name.generate_with_ignorable_key(var.name +
                                                         "_slice_reverse"),
            dtype=var.dtype)
        var.block.append_op(
            type="reverse",
            inputs={'X': out},
            outputs={'Out': [reverse_out_var]},
            attrs={'axis': reverse_axis})

        out = reverse_out_var

    return out


@six.add_metaclass(VariableMetaClass)
X
Xin Pan 已提交
802
class Variable(object):
803
    """
J
Jiabin Yang 已提交
804
    **Notes**:
805
        **The constructor of Variable should not be invoked directly.**
J
Jiabin Yang 已提交
806

807 808
        **In Static Graph Mode: Please use** `Block.create_var` **to create a Static variable which has no data until being feed.**

J
Jiabin Yang 已提交
809 810 811
        **In Dygraph Mode: Please use** :ref:`api_fluid_dygraph_to_variable` **to create a dygraph variable with real data**

    In Fluid, every input and output of an OP is a variable. In most
812
    cases, variables are used for holding different kinds of data or training
J
Jiabin Yang 已提交
813 814
    labels. A variable belongs to a :ref:`api_guide_Block_en` . All variable has its own name and
    two variables in different :ref:`api_guide_Block_en` could have the same name.
815

816
    There are many kinds of variables. Each kind of them has its own attributes
J
Jiabin Yang 已提交
817
    and usages. Please refer to the `framework.proto <https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/framework/framework.proto>`_ for details.
818

T
tianshuo78520a 已提交
819
    Most of a Variable's member variables can be set to be None. It mean
820
    it is not available or will be specified later.
821

822
    Examples:
823 824
        In Static Graph Mode:

825 826
        .. code-block:: python

827
            import paddle.fluid as fluid
828
            cur_program = fluid.Program()
829 830 831 832
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
J
Jiabin Yang 已提交
833
        In `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_  Mode:
834 835 836 837 838 839 840 841 842

        .. code-block:: python

            import paddle.fluid as fluid
            import numpy as np

            with fluid.dygraph.guard():
                new_variable = fluid.dygraph.to_variable(np.arange(10))

843 844
    """

Y
Yu Yang 已提交
845 846
    def __init__(self,
                 block,
Y
Yu Yang 已提交
847
                 type=core.VarDesc.VarType.LOD_TENSOR,
Y
Yu Yang 已提交
848 849 850 851
                 name=None,
                 shape=None,
                 dtype=None,
                 lod_level=None,
852
                 capacity=None,
Q
QI JUN 已提交
853
                 persistable=None,
F
fengjiayi 已提交
854
                 error_clip=None,
Y
Yu Yang 已提交
855
                 stop_gradient=False,
F
fengjiayi 已提交
856
                 is_data=False,
H
Huihuang Zheng 已提交
857
                 need_check_feed=False,
H
hong 已提交
858
                 belong_to_optimizer=False,
Y
Yu Yang 已提交
859
                 **kwargs):
Y
Yu Yang 已提交
860 861
        self.block = block
        if name is None:
Y
Yu Yang 已提交
862
            name = unique_name.generate('_generated_var')
D
Dong Zhihong 已提交
863

Y
Yu Yang 已提交
864
        if dtype is not None:
865
            if not isinstance(dtype, core.VarDesc.VarType):
866
                dtype = convert_np_dtype_to_dtype_(dtype)
867

H
hong 已提交
868 869
        self.belong_to_optimizer = belong_to_optimizer

870 871 872 873 874
        self.error_clip = error_clip

        is_new_var = False
        name = cpt.to_text(name)
        self.desc = self.block.desc.find_var(cpt.to_bytes(name))
875

876 877 878
        if self.desc is None:
            self.desc = self.block.desc.var(cpt.to_bytes(name))
            is_new_var = True
879

880 881 882 883 884 885 886
        if is_new_var:
            self.desc.set_type(type)
        elif self.desc.type() != type:
            raise ValueError("Variable {0} has been created before. The "
                             "previous type is {1}; the new type is {2}. They"
                             " are not matched".format(self.name,
                                                       self.desc.type(), type))
887

888
        if shape is not None:
889
            if is_new_var:
890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
                self.desc.set_shape(shape)
            else:
                old_shape = self.shape
                shape = tuple(shape)
                if shape != old_shape:
                    raise ValueError(
                        "Variable {0} has been created before. the previous "
                        "shape is {1}; the new shape is {2}. They are not "
                        "matched.".format(self.name, old_shape, shape))
        if dtype is not None:
            if is_new_var:
                self.desc.set_dtype(dtype)
            else:
                old_dtype = self.dtype
                if dtype != old_dtype:
                    raise ValueError("Variable {0} has been created before. "
                                     "The previous data type is {1}; the new "
                                     "data type is {2}. They are not "
                                     "matched.".format(self.name, old_dtype,
                                                       dtype))

        if lod_level is not None:
            if is_new_var:
                self.desc.set_lod_level(lod_level)
            else:
                if lod_level != self.lod_level:
                    raise ValueError("Variable {0} has been created before. "
                                     "The previous lod_level is {1}; the new "
                                     "lod_level is {2}. They are not "
                                     "matched".format(self.name, self.lod_level,
                                                      lod_level))
        if persistable is not None:
            if is_new_var:
                self.desc.set_persistable(persistable)
            else:
                if persistable != self.persistable:
                    raise ValueError(
                        "Variable {0} has been created before."
                        "The previous persistable is {1}; the new "
                        "persistable is {2}. They are not matched".format(
                            self.name, self.persistable, persistable))
931

932 933
        if need_check_feed and is_new_var:
            self.desc.set_need_check_feed(need_check_feed)
H
Huihuang Zheng 已提交
934

935 936 937 938 939 940 941
        if capacity is not None:
            if is_new_var:
                self.desc.set_capacity(capacity)
            else:
                # TODO(abhinavarora) : Compare with set capacity once,
                # get_capacity is implemented
                pass
942

943 944 945 946
        self.block.vars[name] = self
        self.op = None
        self._stop_gradient = stop_gradient
        self.is_data = is_data
Y
Yu Yang 已提交
947

948
    @dygraph_only
949 950
    def detach(self):
        """
J
Jiabin Yang 已提交
951
        **Notes**:
T
tianshuo78520a 已提交
952
            **This API is ONLY available in Dygraph mode**
953

954
        Returns a new Variable, detached from the current graph.
955

956
        Returns:
J
Jiabin Yang 已提交
957
             ( :ref:`api_guide_Variable_en` | dtype is same as current Variable): The detached Variable.
958

959

960 961 962 963 964
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
965
                from paddle.fluid.dygraph import Linear
966 967 968 969
                import numpy as np

                data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
                with fluid.dygraph.guard():
970
                    linear = Linear(32, 64)
971
                    data = to_variable(data)
972
                    x = linear(data)
973 974 975
                    y = x.detach()

        """
976
        pass
977

978
    @dygraph_only
979
    def numpy(self):
980
        """
J
Jiabin Yang 已提交
981
        **Notes**:
T
tianshuo78520a 已提交
982
            **This API is ONLY available in Dygraph mode**
983

J
Jiabin Yang 已提交
984
        Returns a numpy array shows the value of current :ref:`api_guide_Variable_en`
985 986 987 988 989

        Returns:
            ndarray: The numpy value of current Variable.

        Returns type:
J
Jiabin Yang 已提交
990
            ndarray: dtype is same as current Variable
991 992 993 994 995 996

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
997
                from paddle.fluid.dygraph import Linear
998 999 1000 1001
                import numpy as np

                data = np.random.uniform(-1, 1, [30, 10, 32]).astype('float32')
                with fluid.dygraph.guard():
1002
                    linear = Linear(32, 64)
1003
                    data = to_variable(data)
1004
                    x = linear(data)
1005 1006 1007
                    print(x.numpy())

        """
1008
        pass
1009

1010 1011 1012
    @dygraph_only
    def set_value(self, value):
        """
J
Jiabin Yang 已提交
1013
        **Notes**:
T
tianshuo78520a 已提交
1014
            **This API is ONLY available in Dygraph mode**
J
Jiabin Yang 已提交
1015

1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
        Set a new value for this Variable.

        Args:
            value (Variable|np.ndarray): the new value.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                from paddle.fluid.dygraph.base import to_variable
1026
                from paddle.fluid.dygraph import Linear
1027 1028
                import numpy as np

1029
                data = np.ones([3, 1024], dtype='float32')
1030
                with fluid.dygraph.guard():
1031
                    linear = fluid.dygraph.Linear(1024, 4)
1032
                    t = to_variable(data)
1033
                    linear(t)  # call with default weight
1034
                    custom_weight = np.random.randn(1024, 4).astype("float32")
1035 1036
                    linear.weight.set_value(custom_weight)  # change existing weight
                    out = linear(t)  # call with different weight
1037 1038

        """
1039
        pass
1040

1041
    @dygraph_only
1042
    def backward(self, backward_strategy=None):
1043
        """
J
Jiabin Yang 已提交
1044
        **Notes**:
T
tianshuo78520a 已提交
1045
            **This API is ONLY available in Dygraph mode**
1046 1047 1048

        Run backward of current Graph which starts from current Variable

J
Jiabin Yang 已提交
1049 1050
        Args:
            backward_strategy( :ref:`api_fluid_dygraph_BackwardStrategy` ): The Backward Strategy to run backward
1051

J
Jiabin Yang 已提交
1052 1053
        Returns:
            NoneType: None
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                        tmp = fluid.dygraph.base.to_variable(x)
J
Jiabin Yang 已提交
1066 1067
                        # if we don't set tmp's stop_gradient as False then, all path to loss will has no gradient since
                        # there is no one need gradient on it.
1068 1069 1070 1071 1072 1073 1074 1075 1076
                        tmp.stop_gradient=False
                        inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
                    backward_strategy = fluid.dygraph.BackwardStrategy()
                    backward_strategy.sort_sum_gradient = True
                    loss2.backward(backward_strategy)

        """
1077
        pass
1078

1079
    @dygraph_only
1080
    def gradient(self):
1081
        """
J
Jiabin Yang 已提交
1082
        **Notes**:
T
tianshuo78520a 已提交
1083
            **This API is ONLY available in Dygraph mode**
1084 1085 1086

        Get the Gradient of Current Variable

J
Jiabin Yang 已提交
1087
        Returns:
1088
            ndarray or tuple of ndarray: if Variable's type is LoDTensor, return numpy value of the gradient of current Variable, if Variable's type is SelectedRows, return tuple of ndarray, first element of tuple is numpy value of the gradient of current Variable, second element of tuple is numpy value of the rows of current Variable.
1089 1090 1091 1092 1093 1094 1095

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

1096
                # example1: return ndarray
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                        tmp = fluid.dygraph.base.to_variable(x)
                        tmp.stop_gradient=False
                        inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
                    backward_strategy = fluid.dygraph.BackwardStrategy()
                    backward_strategy.sort_sum_gradient = True
                    loss2.backward(backward_strategy)
                    print(loss2.gradient())

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123
                # example2: return tuple of ndarray
                with fluid.dygraph.guard():
                    embedding = fluid.dygraph.Embedding(
                        size=[20, 32],
                        param_attr='emb.w',
                        is_sparse=True)
                    x_data = np.arange(12).reshape(4, 3).astype('int64')
                    x_data = x_data.reshape((-1, 3, 1))
                    x = fluid.dygraph.base.to_variable(x_data)
                    out = embedding(x)
                    out.backward()
                    print(embedding.weight.gradient())

1124
        """
1125
        pass
1126

1127
    @dygraph_only
1128
    def clear_gradient(self):
1129
        """
J
Jiabin Yang 已提交
1130
        **Notes**:
T
tianshuo78520a 已提交
1131
            **1. This API is ONLY available in Dygraph mode**
J
Jiabin Yang 已提交
1132 1133

            **2. Use it only Variable has gradient, normally we use this for Parameters since other temporal Variable will be deleted by Python's GC**
1134

J
Jiabin Yang 已提交
1135
        Clear  (set to ``0`` ) the Gradient of Current Variable
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161

        Returns:  None

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    inputs2 = []
                    for _ in range(10):
                        tmp = fluid.dygraph.base.to_variable(x)
                        tmp.stop_gradient=False
                        inputs2.append(tmp)
                    ret2 = fluid.layers.sums(inputs2)
                    loss2 = fluid.layers.reduce_sum(ret2)
                    backward_strategy = fluid.dygraph.BackwardStrategy()
                    backward_strategy.sort_sum_gradient = True
                    loss2.backward(backward_strategy)
                    print(loss2.gradient())
                    loss2.clear_gradient()
                    print("After clear {}".format(loss2.gradient()))

        """
1162
        pass
X
Xin Pan 已提交
1163

1164
    def __str__(self):
Y
Yang Yang(Tony) 已提交
1165 1166
        return self.to_string(True)

F
update  
fengjiayi 已提交
1167
    def to_string(self, throw_on_error, with_details=False):
1168 1169 1170
        """
        Get debug string.

J
Jiabin Yang 已提交
1171 1172 1173 1174 1175
        Args:

            throw_on_error (bool): True if raise an exception when self is not initialized.

            with_details (bool): more details about variables and parameters (e.g. trainable, optimize_attr, ...) will be printed when with_details is True. Default value is False;
1176

1177 1178
        Returns:
            str: The debug string.
1179 1180 1181 1182 1183

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid
1184

1185 1186 1187 1188 1189
                cur_program = fluid.Program()
                cur_block = cur_program.current_block()
                new_variable = cur_block.create_var(name="X",
                                                    shape=[-1, 23, 48],
                                                    dtype='float32')
1190
                print(new_variable.to_string(True))
J
Jiabin Yang 已提交
1191
                print("=============with detail===============")
1192
                print(new_variable.to_string(True, True))
1193
        """
L
lujun 已提交
1194
        if in_dygraph_mode():
1195
            return
1196

F
update  
fengjiayi 已提交
1197 1198
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
1199
        protostr = self.desc.serialize_to_string()
1200
        proto = framework_pb2.VarDesc.FromString(six.binary_type(protostr))
F
update  
fengjiayi 已提交
1201 1202 1203 1204
        res_str = _debug_string_(proto, throw_on_error)
        if with_details:
            additional_attr = ("error_clip", "stop_gradient")
            for attr_name in additional_attr:
1205 1206 1207
                res_str += "%s: %s\n" % (attr_name,
                                         cpt.to_text(getattr(self, attr_name)))

F
update  
fengjiayi 已提交
1208
        return res_str
1209 1210 1211

    __repr__ = __str__

1212
    @property
1213
    def stop_gradient(self):
J
Jiabin Yang 已提交
1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
        """
        Indicating if we stop gradient from current Variable

        **Notes: This Property has default value as** ``True`` **in** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, while Parameter's default value is False. However, in Static Graph Mode all Variable's default stop_gradient value is** ``False``

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            import numpy as np

            with fluid.dygraph.guard():
                value0 = np.arange(26).reshape(2, 13).astype("float32")
                value1 = np.arange(6).reshape(2, 3).astype("float32")
                value2 = np.arange(10).reshape(2, 5).astype("float32")
1229 1230
                linear = fluid.Linear(13, 5, dtype="float32")
                linear2 = fluid.Linear(3, 3, dtype="float32")
J
Jiabin Yang 已提交
1231 1232 1233
                a = fluid.dygraph.to_variable(value0)
                b = fluid.dygraph.to_variable(value1)
                c = fluid.dygraph.to_variable(value2)
1234 1235
                out1 = linear(a)
                out2 = linear2(b)
J
Jiabin Yang 已提交
1236 1237 1238 1239
                out1.stop_gradient = True
                out = fluid.layers.concat(input=[out1, out2, c], axis=1)
                out.backward()

1240
                assert (linear.weight.gradient() == 0).all()
J
Jiabin Yang 已提交
1241 1242
                assert (out1.gradient() == 0).all()
        """
L
lujun 已提交
1243
        if in_dygraph_mode():
1244
            pass
M
minqiyang 已提交
1245
        else:
1246
            return self._stop_gradient
1247

1248 1249
    @stop_gradient.setter
    def stop_gradient(self, s):
L
lujun 已提交
1250
        if in_dygraph_mode():
1251
            pass
1252
        else:
1253
            self._stop_gradient = s
1254

1255 1256
    @property
    def persistable(self):
J
Jiabin Yang 已提交
1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
        """
        Indicating if we current Variable should be long-term alive


        **Notes: This Property will be deprecated and this API is just to help user understand concept**

            **1. All Variable's persistable is** ``False`` **except Parameters.**

            **2. In** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, this property should not be changed**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("persistable of current Var is: {}".format(new_variable.persistable))
        """
L
lujun 已提交
1278
        if in_dygraph_mode():
1279
            pass
1280 1281
        else:
            return self.desc.persistable()
1282

Y
Yu Yang 已提交
1283 1284
    @persistable.setter
    def persistable(self, p):
L
lujun 已提交
1285
        if in_dygraph_mode():
1286 1287 1288
            logging.warn(
                "There will be no use to set persistable in Dygraph Mode, since "
                "you can just do it by hold it as normal Python variable")
1289 1290
        else:
            self.desc.set_persistable(p)
Y
Yu Yang 已提交
1291

Y
Yu Yang 已提交
1292 1293
    @property
    def name(self):
J
Jiabin Yang 已提交
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
        """
        Indicating name of current Variable

        **Notes: If it has two or more Varaible share the same name in the same** :ref:`api_guide_Block_en` **, it means these Variable will share content in no-** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode. This is how we achieve Parameter sharing**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("name of current Var is: {}".format(new_variable.name))
        """
L
lujun 已提交
1310
        if in_dygraph_mode():
1311
            pass
1312 1313
        else:
            return cpt.to_text(self.desc.name())
Y
Yu Yang 已提交
1314

1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334
    @property
    def grad_name(self):
        """
        Indicating name of the gradient Variable of current Variable.

        **Notes: This is a read-only property. It simply returns name of
          gradient Variable from a naming convention but doesn't guarantee
          the gradient exists.**
       
        Examples:
          .. code-block:: python

          import paddle.fluid as fluid

          x = fluid.data(name="x", shape=[-1, 23, 48], dtype='float32')
          print(x.grad_name) # output is "x@GRAD"

        """
        return self.name + "@GRAD"

T
typhoonzero 已提交
1335 1336
    @name.setter
    def name(self, new_name):
L
lujun 已提交
1337
        if in_dygraph_mode():
1338
            pass
1339 1340
        else:
            self.desc.set_name(new_name)
T
typhoonzero 已提交
1341

Y
Yu Yang 已提交
1342 1343
    @property
    def shape(self):
J
Jiabin Yang 已提交
1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
        """
        Indicating shape of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("shape of current Var is: {}".format(new_variable.shape))

        """
Y
Yu Yang 已提交
1361
        # convert to tuple, make it as same as numpy API.
L
lujun 已提交
1362
        if in_dygraph_mode():
1363
            pass
1364 1365
        else:
            return tuple(self.desc.shape())
Y
Yu Yang 已提交
1366 1367

    @property
F
fengjiayi 已提交
1368
    def dtype(self):
J
Jiabin Yang 已提交
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
        """
        Indicating data type of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("Dtype of current Var is: {}".format(new_variable.dtype))
        """
L
lujun 已提交
1385
        if in_dygraph_mode():
1386
            pass
1387 1388
        else:
            return self.desc.dtype()
Y
Yu Yang 已提交
1389 1390

    @property
1391
    @dygraph_not_support
Y
Yu Yang 已提交
1392
    def lod_level(self):
J
Jiabin Yang 已提交
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
        """
        Indicating ``LoD`` info of current Variable, please refer to  :ref:`api_fluid_LoDTensor_en` to check the meaning
        of ``LoD``

        **Notes**:

            **1. This is a read-only property**

            **2. Don't support this property in** `Dygraph <../../user_guides/howto/dygraph/DyGraph.html>`_ **mode, it's value should be** ``0(int)``

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("LoD Level of current Var is: {}".format(new_variable.lod_level))
        """
L
lujun 已提交
1414
        # TODO(minqiyang): Support lod_level in dygraph mode
H
Hongyu Liu 已提交
1415 1416
        if in_dygraph_mode():
            raise Exception("Dygraph model DO NOT supprt lod")
1417 1418 1419 1420

        if self.type == core.VarDesc.VarType.SELECTED_ROWS:
            raise Exception("SelectedRows DO NOT supprt lod")

1421
        return self.desc.lod_level()
Y
Yu Yang 已提交
1422

Y
Yu Yang 已提交
1423 1424
    @property
    def type(self):
J
Jiabin Yang 已提交
1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
        """
        Indicating Type of current Variable

        **Notes: This is a read-only property**

        Examples:
          .. code-block:: python

            import paddle.fluid as fluid
            cur_program = fluid.Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
            print("Type of current Var is: {}".format(new_variable.type))
        """
L
lujun 已提交
1441
        if in_dygraph_mode():
1442
            pass
1443 1444
        else:
            return self.desc.type()
Y
Yu Yang 已提交
1445

W
Wu Yi 已提交
1446
    def _set_error_clip(self, error_clip):
1447 1448 1449 1450 1451 1452 1453 1454 1455
        """
        Set the error_clip.

        Args:
            error_clip(BaseErrorClipAttr) : The new error_clip.

        Returns:
            None
        """
1456 1457
        self.error_clip = error_clip

1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
    def _set_info(self, key, value):
        """
        Set key-value information for this variable.

        Args:
            key(str): Key for this information.
            value(object): The value associated to the key.

        Returns: 
            None
        """
        if not hasattr(self, "_info"):
            self._info = {}
        self._info[key] = value

    def _get_info(self, key):
        """
        Get the information of this variable corresponding to key.

        Args:
            key(str): Key for this information.

        Returns: 
            object
        """
        if hasattr(self, "_info") and key in self._info:
            return self._info[key]
        return None

1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497
    def _slice_indices(self, slice, length):
        """
        Reference implementation for the slice.indices method.
        """
        # Compute step and length as integers.
        step = 1 if slice.step is None else slice.step

        # Raise ValueError for negative length or zero step.
        if length < 0:
            raise ValueError("length should not be negative")
        if step == 0:
T
tianshuo78520a 已提交
1498
            raise ValueError("slice step can not be zero")
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573

        # Find lower and upper bounds for start and stop.
        lower = -1 if step < 0 else 0
        upper = length - 1 if step < 0 else length

        # Compute start.
        if slice.start is None:
            start = upper if step < 0 else lower
        else:
            start = slice.start
            start = max(start + length, lower) if start < 0 else min(start,
                                                                     upper)

        # Compute stop.
        if slice.stop is None:
            stop = lower if step < 0 else upper
        else:
            stop = slice.stop
            stop = max(stop + length, lower) if stop < 0 else min(stop, upper)

        return start, stop, step

    def _detectEllipsis(self, item):
        has_ellipsis = False
        start = 0
        end = len(self.shape)
        for index, o in enumerate(item):
            if o is Ellipsis:
                if has_ellipsis:
                    raise ValueError("Index can have one ellipsis only.")
                has_ellipsis = True
                start = index
            else:
                if has_ellipsis:
                    end = index
        return has_ellipsis, start, end

    def _reconstructSliceinfo(self, item):
        has_ellipsis, start, end = self._detectEllipsis(item)
        if has_ellipsis:
            newitem = []
            for i in range(start):
                newitem.append(item[i])
            for i in range(start, end):
                newitem.append(slice(None, None, None))
            for i in range(end, len(item)):
                newitem.append(item[i])
            return newitem
        else:
            return None

    def _detectContinuesSlice(self, item):
        starts = []
        ends = []
        for index, o in enumerate(item):
            if isinstance(o, int):
                start = int(o)
                if (index > 0 and index >= self.shape[index]) \
                        or (index < 0 and (index + self.shape[index]) < 0):
                    raise IndexError("invalid index")
                start = max(start + self.shape[index], 0) if start < 0 else min(
                    start, self.shape[index])
                starts.append(start)
                ends.append(start + 1)
            elif isinstance(o, slice):
                start, stop, step = self._slice_indices(o, self.shape[index])
                if step == 1 or step == -1:
                    starts.append(start)
                    ends.append(stop)
                else:
                    return False, None
            else:
                raise IndexError("Valid index accept int or slice or ellipsis")
        return True, [starts, ends]

L
lujun 已提交
1574
    def _cloneVar(self, copy=False):
1575 1576
        if not copy:
            return self.block.create_var(
H
Hongyu Liu 已提交
1577 1578
                name=unique_name.generate_with_ignorable_key(self.name),
                dtype=self.dtype)
1579 1580 1581 1582
        else:
            return self

    def _sliceVar(self, axes, starts, ends):
L
lujun 已提交
1583
        new_var = self._cloneVar()
1584 1585 1586 1587 1588 1589 1590 1591 1592 1593
        self.block.append_op(
            type="slice",
            inputs={'Input': [self]},
            outputs={'Out': [new_var]},
            attrs={'axes': axes,
                   'starts': starts,
                   'ends': ends})
        return new_var

    def _concatVar(self, inputs, axis):
L
lujun 已提交
1594
        new_var = self._cloneVar()
1595 1596 1597 1598 1599 1600 1601 1602 1603 1604
        self.block.append_op(
            type="concat",
            inputs={'X': inputs},
            outputs={'Out': [new_var]},
            attrs={'axis': axis, })
        return new_var

    def _sliceAndConcatVar(self, item, axis):
        if isinstance(item, slice):
            if self.shape[axis] < 0:
L
lujun 已提交
1605
                return self._cloneVar(True)
1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
            start, stop, step = self._slice_indices(item, self.shape[axis])
            if step == 1:
                return self._sliceVar([axis], [start], [stop])
            else:
                vars = []
                if step > 0:
                    while start < stop:
                        vars.append(
                            self._sliceVar([axis], [start], [start + 1]))
                        start += step
                else:
                    while start > stop:
                        vars.append(
                            self._sliceVar([axis], [start], [start + 1]))
                        start += step
                return self._concatVar(vars, axis)
        elif isinstance(item, int):
            if self.shape[axis] < 0:
L
lujun 已提交
1624
                return self._cloneVar(True)
1625
            index = int(item)
1626
            if (index > 0 and index >= self.shape[axis]) \
1627 1628 1629 1630 1631 1632 1633
                    or (index < 0 and (index + self.shape[axis]) < 0):
                raise IndexError("invalid index")
            return self._sliceVar([axis], [index], [index + 1])
        else:
            raise IndexError("Valid index accept int or slice or tuple")

    def __getitem__(self, item):
1634
        return _getitem_impl_(self, item)
1635

Y
Yu Yang 已提交
1636

F
fengjiayi 已提交
1637 1638 1639
def get_all_op_protos():
    """
    Get all registered op proto from PaddlePaddle C++ end.
1640

1641 1642
    Returns:
       list: list of OpProto.
F
fengjiayi 已提交
1643 1644 1645 1646
    """
    protostrs = core.get_all_op_protos()
    ret_values = []
    for pbstr in protostrs:
1647
        op_proto = framework_pb2.OpProto.FromString(six.binary_type(pbstr))
F
fengjiayi 已提交
1648 1649 1650 1651 1652
        ret_values.append(op_proto)
    return ret_values


class OpProtoHolder(object):
1653 1654 1655 1656
    """
    A global variable to hold all OpProtos from C++ as a map
    """

F
fengjiayi 已提交
1657 1658 1659 1660 1661 1662 1663 1664 1665
    @classmethod
    def instance(cls):
        if not hasattr(cls, '_instance'):
            cls._instance = cls()
        return cls._instance

    def __init__(self):
        assert not hasattr(
            self.__class__,
1666
            '_instance'), 'Please use `instance()` to get OpProtoHolder object!'
F
fengjiayi 已提交
1667 1668 1669 1670 1671 1672
        op_protos = get_all_op_protos()
        self.op_proto_map = {}
        for proto in op_protos:
            self.op_proto_map[proto.type] = proto

    def get_op_proto(self, type):
1673 1674 1675 1676 1677 1678 1679 1680
        """
        Get OpProto by a type string.
        Args:
            type(str): The type that operator registered in C++ side.

        Returns(framework_pb2.OpProto): The OpProto

        """
Y
Yu Yang 已提交
1681 1682
        if type not in self.op_proto_map:
            raise ValueError("Operator \"%s\" has not been registered." % type)
F
fengjiayi 已提交
1683 1684
        return self.op_proto_map[type]

1685 1686 1687 1688 1689 1690
    def update_op_proto(self):
        op_protos = get_all_op_protos()
        for proto in op_protos:
            if proto.type not in self.op_proto_map:
                self.op_proto_map[proto.type] = proto

1691 1692 1693 1694
    @staticmethod
    def generated_op_attr_names():
        return {
            core.op_proto_and_checker_maker.kOpRoleAttrName(),
S
sneaxiy 已提交
1695
            core.op_proto_and_checker_maker.kOpRoleVarAttrName(),
1696 1697
            core.op_proto_and_checker_maker.kOpNameScopeAttrName(),
            core.op_proto_and_checker_maker.kOpCreationCallstackAttrName()
1698 1699
        }

F
fengjiayi 已提交
1700

X
Xin Pan 已提交
1701
class Operator(object):
1702
    """
1703 1704 1705 1706 1707 1708 1709
    In Fluid, all the operation are represented by Operator, and Operator
    is regarded as a build in an instruction of a Block. Users can use the
    build in instructions to describe their neural network.

    Args:
        block(Block): The block has the current operator.
        desc(core.OpDesc): The protobuf description of Operator.
C
chengduoZH 已提交
1710
        type(str): The type of operator. Default None.
1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730
        inputs(dict): The input of this Operator. it is a dictionary, for every
            element, key is the input parameter name, and value is a list of
            variables. Default None.
        outputs(dict): The output of this Operator. it is a dictionary, for
            every element, key is the input parameter name, and value is a list
            of variables. Default None.
        attrs(dict): The attributes of this Operator. it is a dictionary, for
            every element, key is attribute name, and value is the attribute value.
            The attribute type should be as same as the type registered in C++ side.
            Default None.

    Returns:
        Operator: The initialized Operator.

    Raises:
        ValueError: If the passed input, output and attrs doesn't match the
            initializing Operator's that registered in C++ side.

    Notes:
        The constructor of operator should not be invoked directly. Use
W
Wu Yi 已提交
1731
        Block.append_op or Block._prepend_op instead.
1732 1733 1734 1735

    Examples:
        .. code-block:: python

1736
            import paddle.fluid as fluid
1737
            cur_program = fluid.Program()
1738 1739 1740 1741 1742
            cur_block = cur_program.current_block()
            # var1 += var2 + var3
            cur_block.append_op(type="sum",
                                inputs={"X": [var1, var2, var3]},
                                outputs={"Out": [var1]})
1743
    """
1744
    OP_WITHOUT_KERNEL_SET = {
1745 1746
        'feed', 'fetch', 'recurrent', 'go', 'rnn_memory_helper_grad',
        'conditional_block', 'while', 'send', 'recv', 'listen_and_serv',
1747 1748
        'fl_listen_and_serv', 'ncclInit', 'select', 'checkpoint_notify',
        'gen_nccl_id', 'c_gen_nccl_id', 'c_comm_init', 'c_sync_calc_stream',
1749
        'c_sync_comm_stream'
1750
    }
1751

Y
Yu Yang 已提交
1752 1753
    def __init__(self,
                 block,
Y
Yu Yang 已提交
1754
                 desc,
Y
Yu Yang 已提交
1755 1756 1757
                 type=None,
                 inputs=None,
                 outputs=None,
M
minqiyang 已提交
1758
                 attrs=None):
L
lujun 已提交
1759
        if in_dygraph_mode():
1760 1761
            if type is None:
                raise ValueError(
1762
                    "`type` to initialized an Operator can not be None.")
J
Jiabin Yang 已提交
1763
            self._type = type
M
minqiyang 已提交
1764
            self.attrs = attrs if attrs else {}
1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
        else:
            self.block = block
            self.desc = desc
            # note: not add self.attrs here:
            # https://github.com/PaddlePaddle/Paddle/pull/12583#pullrequestreview-145093173
            op_attrs = attrs
            if op_attrs is None:
                op_attrs = dict()
            del attrs

            op_maker = core.op_proto_and_checker_maker

            if op_maker.kOpRoleAttrName() not in op_attrs:
                op_attrs[op_maker.kOpRoleAttrName(
1779
                )] = self.block.program._op_role
1780 1781 1782

            role_var_name = op_maker.kOpRoleVarAttrName()
            if len(self.block.program.
1783 1784
                   _op_role_var) != 0 and role_var_name not in op_attrs:
                op_attrs[role_var_name] = self.block.program._op_role_var
1785 1786 1787 1788 1789 1790 1791 1792

            if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0:
                del op_attrs[role_var_name]

            if len(self.desc.type()) != 0:
                return
            if type is None:
                raise ValueError(
1793
                    "`type` to initialized an Operator can not be None.")
1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824
            else:
                callstack_var_name = op_maker.kOpCreationCallstackAttrName()
                op_attrs[callstack_var_name] = list(
                    reversed(traceback.format_stack()))[1:]

            self.desc.set_type(type)
            proto = OpProtoHolder.instance().get_op_proto(type)

            namescope_var_name = op_maker.kOpNameScopeAttrName()
            op_attrs[namescope_var_name] = _full_name_scope()

            def find_name(var_list, name):
                for var_name in var_list:
                    if var_list[var_name] is not None and var_name == name:
                        return True
                return False

            if inputs is not None:
                for in_proto in proto.inputs:
                    found = find_name(inputs, in_proto.name)
                    assert found or in_proto.dispensable, "Input {} not found".format(
                        in_proto.name)
                    if found:
                        in_args = inputs[in_proto.name]
                        if not isinstance(in_args, list):
                            in_args = [in_args]
                        if not in_proto.duplicable and len(in_args) > 1:
                            raise ValueError(
                                "Input %s expects only one input, but %d are given."
                                % (in_proto.name, len(in_args)))
                        in_arg_names = []
1825
                        for index, arg in enumerate(in_args):
1826 1827 1828 1829
                            if isinstance(arg, six.string_types):
                                in_arg_names.append(arg)
                            elif isinstance(arg, six.binary_type):
                                in_arg_names.append(arg.decode())
1830
                            elif isinstance(arg, Variable):
1831
                                in_arg_names.append(cpt.to_text(arg.name))
1832
                            else:
1833 1834 1835 1836
                                raise TypeError(
                                    "The type of '%s' in operator %s should be "
                                    "one of [basestring(), str, Varibale] in python2, "
                                    "or one of [str, bytes, Variable] in python3."
1837 1838
                                    "but received : %s" %
                                    (in_proto.name, type, arg))
1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864
                        self.desc.set_input(in_proto.name, in_arg_names)
                    else:
                        self.desc.set_input(in_proto.name, [])

            if outputs is not None:
                for m in proto.outputs:
                    if (m.name not in outputs) and m.dispensable:
                        continue
                    if not ((m.name in outputs) or m.dispensable):
                        raise ValueError(("Incorrect setting for output(s) of "
                                          "operator \"%s\", should set: [%s].")
                                         % (type, m.name))
                for out_proto in proto.outputs:
                    if out_proto.name not in outputs:
                        continue
                    out_args = outputs[out_proto.name]
                    if not isinstance(out_args, list):
                        out_args = [out_args]
                    if not out_proto.duplicable and len(out_args) > 1:
                        raise ValueError(
                            "Output %s expects only one output, but %d are given."
                            % (out_proto.name, len(out_args)))
                    out_arg_names = []
                    for arg in out_args:
                        out_arg_names.append(cpt.to_text(arg.name))
                        # TODO(minqiyang): could we remove variable's op in static mode?
L
lujun 已提交
1865
                        if not in_dygraph_mode():
1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884
                            arg.op = self
                    self.desc.set_output(out_proto.name, out_arg_names)

            if op_attrs is not None:
                if not isinstance(op_attrs, dict):
                    raise TypeError("'attrs' should be a dict.")
                for attr in proto.attrs:
                    attr_name = attr.name
                    if (attr_name not in op_attrs) or (
                            op_attrs[attr_name] is None):
                        continue
                    attr_val = op_attrs[attr_name]
                    self._update_desc_attr(attr_name, attr_val)

            self.desc.check_attrs()
            if self._has_kernel(type):
                self.desc.infer_var_type(self.block.desc)
                self.desc.infer_shape(self.block.desc)

W
Wu Yi 已提交
1885
    def _has_kernel(self, op_type):
1886 1887
        return op_type not in self.OP_WITHOUT_KERNEL_SET

Y
Yang Yang(Tony) 已提交
1888
    def to_string(self, throw_on_error):
1889
        """
1890 1891
        Get debug string.

1892
        Args:
1893 1894
            throw_on_error(bool): Whether to raise exception if self is not
                initialized.
1895

1896 1897
        Returns:
            str: The debug string.
1898 1899

        """
1900
        protostr = self.desc.serialize_to_string()
1901
        proto = framework_pb2.OpDesc.FromString(six.binary_type(protostr))
Y
Yang Yang(Tony) 已提交
1902 1903 1904 1905
        return _debug_string_(proto, throw_on_error)

    def __str__(self):
        return self.to_string(True)
1906 1907 1908

    __repr__ = __str__

F
fengjiayi 已提交
1909 1910
    @property
    def type(self):
L
lujun 已提交
1911
        if in_dygraph_mode():
J
Jiabin Yang 已提交
1912
            return self._type
1913 1914
        else:
            return self.desc.type()
F
fengjiayi 已提交
1915 1916

    def input(self, name):
1917
        """
1918
        Get the input arguments according to the input parameter name.
1919

1920 1921
        Args:
            name(str): The input parameter name.
1922

1923 1924 1925
        Returns:
            list: return the list of argument names that associated with \
                the specific parameter name.
1926
        """
F
fengjiayi 已提交
1927 1928
        return self.desc.input(name)

W
Wu Yi 已提交
1929
    def _rename_input(self, old_name, new_name):
1930 1931 1932 1933 1934 1935 1936 1937 1938 1939
        """
        Rename the `old_name` to `new_name`.

        Args:
            old_name(str): The old name of the Operator's input.
            new_name(str): The new name of the Operator's input.

        Returns:
            None
        """
W
Wu Yi 已提交
1940
        self.desc._rename_input(old_name, new_name)
T
typhoonzero 已提交
1941

W
Wu Yi 已提交
1942
    def _rename_output(self, old_name, new_name):
1943 1944 1945 1946 1947 1948 1949 1950 1951 1952
        """
        Rename the `old_name` to `new_name`.

        Args:
            old_name(str): The old name of the Operator's output.
            new_name(str): The new name of the Operator's output.

        Returns:
            None
        """
W
Wu Yi 已提交
1953
        self.desc._rename_output(old_name, new_name)
T
typhoonzero 已提交
1954

F
fengjiayi 已提交
1955 1956 1957 1958
    @property
    def input_names(self):
        return self.desc.input_names()

T
typhoonzero 已提交
1959 1960 1961 1962 1963 1964 1965 1966
    @property
    def input_arg_names(self):
        return self.desc.input_arg_names()

    @property
    def output_arg_names(self):
        return self.desc.output_arg_names()

F
fengjiayi 已提交
1967
    def output(self, name):
1968
        """
1969
        Get output arguments by the output parameter name.
1970

1971 1972
        Args:
            name(str): The output parameter name.
1973

1974 1975 1976
        Returns:
            list: return the list of argument names associated with \
                the specific parameter name.
1977
        """
F
fengjiayi 已提交
1978 1979 1980 1981 1982 1983
        return self.desc.output(name)

    @property
    def output_names(self):
        return self.desc.output_names()

1984 1985 1986 1987 1988 1989 1990 1991
    @property
    def idx(self):
        for i, op in enumerate(self.block.ops):
            if op == self:
                return i
        raise ValueError(
            "Can't find op itself in it's block. It could be a bug of Paddle.")

F
fengjiayi 已提交
1992
    def has_attr(self, name):
1993
        """
1994 1995
        Whether this Operator has the attribute with name or not.

1996
        Args:
1997
            name(str): the attribute name.
1998

1999 2000
        Returns:
            bool: True if has this attribute.
2001 2002

        """
F
fengjiayi 已提交
2003 2004 2005
        return self.desc.has_attr(name)

    def attr_type(self, name):
2006
        """
2007
        Get the type of attribute by attribute's name.
2008

2009 2010
        Args:
            name(str): the attribute name.
2011

2012 2013
        Returns:
            core.AttrType: the attribute type.
2014
        """
F
fengjiayi 已提交
2015 2016
        return self.desc.attr_type(name)

W
Wu Yi 已提交
2017
    def _set_attr(self, name, val):
2018 2019 2020 2021 2022 2023 2024 2025 2026 2027
        """
        Set the value of attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.

        Raises:
            ValueError: If the type of value doesn't match with desc.attr_type(name).
        """
G
gongweibao 已提交
2028 2029
        self._update_desc_attr(name, val)

2030 2031 2032
    def _remove_attr(self, name):
        self.desc.remove_attr(name)

G
gongweibao 已提交
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
    def _update_desc_attr(self, name, val):
        """
        Update the value of desc's attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.

        Raises:
            ValueError: If the type of value doesn't match with desc.attr_type(name).
        """
Q
Qiyang Min 已提交
2044 2045
        if isinstance(val, Block):
            self.desc.set_block_attr(name, val.desc)
Y
Yancey1989 已提交
2046 2047
        elif isinstance(val, list) and val and all(
                isinstance(v, Block) for v in val):
2048
            self.desc.set_blocks_attr(name, [v.desc for v in val])
Q
Qiyang Min 已提交
2049 2050 2051 2052
        elif isinstance(val, core.BlockDesc) or \
                isinstance(val, core.ProgramDesc):
            self.desc.set_serialized_attr(name, val.serialize_to_string())
        else:
W
Wu Yi 已提交
2053
            self.desc._set_attr(name, val)
Y
yuyang18 已提交
2054

F
fengjiayi 已提交
2055 2056 2057 2058 2059
    @property
    def attr_names(self):
        return self.desc.attr_names()

    def attr(self, name):
2060
        """
2061 2062
        Get the attribute by name.

2063
        Args:
2064
            name(str): the attribute name.
2065

2066 2067
        Returns:
            bool|int|str|float|list: The attribute value. The return value
2068 2069
            can be any valid attribute type.
        """
F
fengjiayi 已提交
2070
        return self.desc.attr(name)
Y
Yu Yang 已提交
2071

W
Wu Yi 已提交
2072
    def _block_attr_id(self, name):
2073
        """
G
gongweibao 已提交
2074
        Get the block attribute's id by name.
2075

2076 2077
        Args:
            name(str): the attribute name.
2078

2079 2080
        Returns:
            int: the block index.
2081
        """
W
Wu Yi 已提交
2082
        return self.desc._block_attr_id(name)
G
gongweibao 已提交
2083

W
Wu Yi 已提交
2084
    def _block_attr(self, name):
G
gongweibao 已提交
2085 2086 2087 2088 2089 2090 2091 2092 2093 2094
        """
        Get the block attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            block: the block attribute.
        """

W
Wu Yi 已提交
2095
        id = self._block_attr_id(name)
G
gongweibao 已提交
2096 2097 2098
        assert (id >= 0 and id < len(self.block.program.blocks))
        return self.block.program.blocks[id]

W
Wu Yi 已提交
2099
    def _blocks_attr(self, name):
G
gongweibao 已提交
2100 2101 2102 2103 2104 2105 2106 2107 2108 2109
        """
        Get the blocks attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            list: list of the blocks attribute.
        """
        attrs = []
W
Wu Yi 已提交
2110
        for i in self._blocks_attr_ids(name):
G
gongweibao 已提交
2111 2112 2113 2114 2115
            assert (i >= 0 and i < len(self.block.program.blocks))
            attrs.append(self.block.program.blocks[i])

        return attrs

W
Wu Yi 已提交
2116
    def _blocks_attr_ids(self, name):
G
gongweibao 已提交
2117 2118 2119 2120 2121 2122 2123 2124 2125 2126
        """
        Get the blocks attribute's ids by name.

        Args:
            name(str): the attribute name.

        Returns:
            list: list of the blocks ids.
        """

W
Wu Yi 已提交
2127
        return self.desc._blocks_attr_ids(name)
Y
Yu Yang 已提交
2128

J
JiayiFeng 已提交
2129
    def all_attrs(self):
F
fengjiayi 已提交
2130
        """
2131 2132 2133
        Get the attribute dict.

        Returns:
G
gongweibao 已提交
2134
            dict: The Operator's attribute dict, name->attr.
F
fengjiayi 已提交
2135 2136 2137 2138
        """
        attr_names = self.attr_names
        attr_map = {}
        for n in attr_names:
G
gongweibao 已提交
2139 2140
            attr_type = self.desc.attr_type(n)
            if attr_type == core.AttrType.BLOCK:
W
Wu Yi 已提交
2141
                attr_map[n] = self._block_attr(n)
G
gongweibao 已提交
2142 2143 2144
                continue

            if attr_type == core.AttrType.BLOCKS:
W
Wu Yi 已提交
2145
                attr_map[n] = self._blocks_attr(n)
G
gongweibao 已提交
2146 2147 2148 2149
                continue

            attr_map[n] = self.attr(n)

F
fengjiayi 已提交
2150 2151
        return attr_map

Y
Yu Yang 已提交
2152

Y
Yu Yang 已提交
2153
class Block(object):
2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167
    """
    In Fluid, a Program is consistence of multi-Block, and Block stores
    VarDesc and OpDesc. In a specific Block, a VarDesc have a unique name.
    One block could have some child blocks, and child block's name scopes
    should inherit the parent's so that OpDesc in child block can reference
    a VarDesc that is stored in the parent block.
    Please reference the framework.proto for details.

    Args:
        program(Program): The Program that the Block belongs to.
        idx(int): The block's id in the Program.

    Notes:
        The constructor of Block should not be invoked directly. Please
W
Wu Yi 已提交
2168
        use `Program._create_block()` to create a block.
2169 2170 2171 2172

    Examples:
        .. code-block:: python

2173 2174 2175
            import paddle.fluid as fluid

            cur_program = fluid.Program()
2176 2177 2178 2179 2180 2181 2182 2183 2184
            cur_block = cur_program.current_block()
            var = cur_block.create_var(name="X",
                                       shape=[-1, 23, 48],
                                       dtype='float32')
            cur_block.append_op(type="abs",
                                inputs={"X": [var]},
                                outputs={"Out": [var]})
    """

Y
Yu Yang 已提交
2185
    def __init__(self, program, idx):
Y
Yu Yang 已提交
2186
        self.desc = program.desc.block(idx)
2187
        self.vars = collections.OrderedDict()  # var_name --> var
Q
qiaolongfei 已提交
2188
        self.ops = list()  # operator list
Y
Yu Yang 已提交
2189
        self.program = program
2190
        self.removed_vars = collections.OrderedDict()
Y
Yu Yang 已提交
2191

2192
    def __str__(self):
Y
Yang Yang(Tony) 已提交
2193 2194
        return self.to_string(True)

F
fengjiayi 已提交
2195 2196
    def to_string(self, throw_on_error, with_details=False):
        """
2197 2198
        Get debug string.

F
fengjiayi 已提交
2199 2200
        Args:
            throw_on_error(bool): raise exception when self is not initialized
2201
                when throw_on_error is True.
F
update  
fengjiayi 已提交
2202
            with_details(bool): more details about variables and parameters
2203 2204
                (e.g. trainable, optimize_attr, ...) will be printed when
                with_details is True. Default False.
F
fengjiayi 已提交
2205

2206 2207
        Returns:
            str: The debug string.
F
fengjiayi 已提交
2208 2209 2210 2211
        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
F
fengjiayi 已提交
2212
            re_add_indent = re.compile(r"\n(.)")
F
fengjiayi 已提交
2213 2214
            res_str = "blocks {\n  idx: %d\n  parent_idx: %d" % (
                self.idx, self.parent_idx)
2215
            for var in list(self.vars.values()):
F
fengjiayi 已提交
2216
                res_str += "\n  vars {\n    %s  }" % re_add_indent.sub(
F
update  
fengjiayi 已提交
2217
                    r"\n    \1", var.to_string(throw_on_error, with_details))
F
fengjiayi 已提交
2218
            for op in self.ops:
F
fengjiayi 已提交
2219 2220
                res_str += "\n  ops {\n    %s  }" % re_add_indent.sub(
                    r"\n    \1", op.to_string(throw_on_error))
F
fengjiayi 已提交
2221 2222 2223
            res_str += "\n}"
        else:
            protostr = self.desc.serialize_to_string()
2224 2225
            proto = framework_pb2.BlockDesc.FromString(
                six.binary_type(protostr))
F
fengjiayi 已提交
2226 2227
            res_str = _debug_string_(proto, throw_on_error)
        return res_str
2228 2229 2230

    __repr__ = __str__

Y
Yu Yang 已提交
2231 2232
    @property
    def parent_idx(self):
Y
Yu Yang 已提交
2233
        return self.desc.parent
Y
Yu Yang 已提交
2234

Y
Yu Yang 已提交
2235 2236 2237 2238
    @property
    def forward_block_idx(self):
        return self.desc.get_forward_block_idx()

W
Wu Yi 已提交
2239
    def _set_forward_block_idx(self, idx):
2240 2241 2242 2243 2244 2245 2246 2247 2248
        """
        Set the forward block Idx.

        Args:
            idx(int): the block index.

        Returns:
            None
        """
W
Wu Yi 已提交
2249
        self.desc._set_forward_block_idx(idx)
Y
Yu Yang 已提交
2250

2251 2252 2253 2254 2255 2256 2257 2258
    @property
    def backward_block_idx(self):
        cur_block_idx = self.idx
        for block in self.program.blocks:
            if block.forward_block_idx == cur_block_idx:
                return block.idx
        return -1

Y
Yu Yang 已提交
2259 2260
    @property
    def idx(self):
Y
Yu Yang 已提交
2261
        return self.desc.id
Y
Yu Yang 已提交
2262

Q
Qiao Longfei 已提交
2263
    def var(self, name):
2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276
        """
        Get a Variable by name from this block.

        Args:
            name(str): the Variable's name.

        Raises:
            ValueError: The If input's type is not str, or this block
                doesn't have a Variable with the giving name.

        Returns:
            Variable: the Variable with the giving name.
        """
2277
        if not isinstance(name, six.string_types):
M
minqiyang 已提交
2278 2279 2280
            raise TypeError(
                "var require string as parameter, but get %s instead." %
                (type(name)))
Y
Yu Yang 已提交
2281 2282
        v = self.vars.get(name, None)
        if v is None:
Q
Qiao Longfei 已提交
2283
            raise ValueError("var %s not in this block" % name)
Y
Yu Yang 已提交
2284
        return v
Q
Qiao Longfei 已提交
2285

X
Xin Pan 已提交
2286
    def _find_var_recursive(self, name):
2287 2288 2289 2290 2291 2292 2293
        """
        Get a Variable by name from this block recursively.

        Args:
            name(str): the Variable's name.

        Returns:
X
Xin Pan 已提交
2294
            Variable: the Variable with the giving name. Or None if not found.
2295
        """
Y
Yu Yang 已提交
2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319
        frontier = list()
        visited = set()

        frontier.append(self)

        prog = self.program

        while len(frontier) != 0:  # BFS
            cur = frontier[0]
            frontier = frontier[1:]

            if id(cur) in visited:
                continue

            if cur.has_var(name):
                return cur.var(name)

            if cur.parent_idx != -1:
                frontier.append(prog.block(cur.parent_idx))

            if cur.forward_block_idx != -1:
                frontier.append(prog.block(cur.forward_block_idx))

            visited.add(id(cur))
X
Xin Pan 已提交
2320
        return None
Y
Yu Yang 已提交
2321

X
Xin Pan 已提交
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340
    def _var_recursive(self, name):
        """
        Get a Variable by name from this block recursively.

        Args:
            name(str): the Variable's name.

        Raises:
            ValueError: this block and this parent block doesn't
                have a Variable with the giving name.

        Returns:
            Variable: the Variable with the giving name.
        """
        var = self._find_var_recursive(name)
        if var:
            return var
        else:
            raise ValueError("Var {0} is not found recursively".format(name))
F
fengjiayi 已提交
2341

Q
Qiao Longfei 已提交
2342
    def all_parameters(self):
2343
        return list(self.iter_parameters())
2344

2345
    def iter_parameters(self):
M
minqiyang 已提交
2346
        return (item[1] for item in six.iteritems(self.vars)
2347
                if isinstance(item[1], Parameter))
Q
Qiao Longfei 已提交
2348

Y
Yu Yang 已提交
2349
    def create_var(self, *args, **kwargs):
L
Leo Chen 已提交
2350 2351 2352
        if in_dygraph_mode():
            var = _varbase_creator(*args, **kwargs)
        else:
2353 2354 2355
            var = Variable(block=self, *args, **kwargs)
            if 'initializer' in kwargs:
                kwargs['initializer'](var, self)
Q
Qiao Longfei 已提交
2356
        return var
Y
Yu Yang 已提交
2357

Q
Qiao Longfei 已提交
2358 2359 2360
    def has_var(self, name):
        return name in self.vars

W
Wu Yi 已提交
2361
    def _rename_var(self, name, new_name):
T
typhoonzero 已提交
2362 2363
        """
        Rename variable in vars and ops' inputs and outputs
2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375

        Args:
            name(str): the name that need to be renamed.
            new_name(str): the name that need to rename to.

        Raises:
            ValueError: If this block doesn't have this the giving name,
                or the type of the var with the giving name is not Parameter
                or Variable.

        Returns:
            Variable: the Variable with the giving name.
T
typhoonzero 已提交
2376
        """
M
minqiyang 已提交
2377 2378
        name = cpt.to_text(name)
        new_name = cpt.to_text(new_name)
M
minqiyang 已提交
2379

T
typhoonzero 已提交
2380
        if not self.has_var(name):
2381
            raise ValueError("var %s is not in current block" % name)
T
wip  
typhoonzero 已提交
2382 2383
        v = self.var(name)
        if type(v) == Parameter:
T
typhoonzero 已提交
2384
            var_type = "Parameter"
T
wip  
typhoonzero 已提交
2385 2386 2387 2388 2389 2390 2391
            stop_gradient = v.stop_gradient
            trainable = v.trainable
            optimize_attr = v.optimize_attr
            regularizer = v.regularizer
            gradient_clip_attr = v.gradient_clip_attr
            error_clip = v.error_clip
        elif type(v) == Variable:
T
typhoonzero 已提交
2392
            var_type = "Variable"
T
wip  
typhoonzero 已提交
2393 2394 2395 2396
            error_clip = v.error_clip
            stop_gradient = v.stop_gradient
        else:
            raise ValueError("unsupported var type: %s", type(v))
T
typhoonzero 已提交
2397
        orig_var_type = v.type
M
minqiyang 已提交
2398
        self.desc._rename_var(cpt.to_bytes(name), cpt.to_bytes(new_name))
W
Wu Yi 已提交
2399
        # NOTE: v is destroyed by C++ after calling _rename_var.
M
minqiyang 已提交
2400
        d = self.desc.find_var(cpt.to_bytes(new_name))
T
typhoonzero 已提交
2401
        if var_type == "Parameter":
L
Leo Chen 已提交
2402 2403
            if in_dygraph_mode():
                var = ParamBase(
2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414
                    d.shape(),
                    d.dtype(),
                    type=orig_var_type,
                    name=new_name,
                    stop_gradient=stop_gradient,
                    trainable=trainable,
                    optimize_attr=optimize_attr,
                    regularizer=regularizer,
                    gradient_clip_attr=gradient_clip_attr,
                    error_clip=error_clip)
            else:
L
Leo Chen 已提交
2415 2416
                var = Parameter(
                    self,
2417 2418 2419 2420 2421 2422 2423 2424 2425 2426
                    d.shape(),
                    d.dtype(),
                    type=orig_var_type,
                    name=new_name,
                    stop_gradient=stop_gradient,
                    trainable=trainable,
                    optimize_attr=optimize_attr,
                    regularizer=regularizer,
                    gradient_clip_attr=gradient_clip_attr,
                    error_clip=error_clip)
T
typhoonzero 已提交
2427
        elif var_type == "Variable":
T
wip  
typhoonzero 已提交
2428 2429
            var = Variable(
                self,
T
typhoonzero 已提交
2430
                type=orig_var_type,
T
wip  
typhoonzero 已提交
2431 2432 2433 2434
                name=new_name,
                error_clip=error_clip,
                stop_gradient=stop_gradient)

W
Wu Yi 已提交
2435
        # rename the python side, _sync_with_cpp will only add
T
wip  
typhoonzero 已提交
2436 2437 2438
        # new vars/ops to python side.
        self.vars[new_name] = var
        del self.vars[name]
W
Wu Yi 已提交
2439
        self._sync_with_cpp()
2440
        return var
T
typhoonzero 已提交
2441

W
Wu Yi 已提交
2442 2443
    def _remove_var(self, name):
        self._sync_with_cpp()
M
minqiyang 已提交
2444
        self.desc._remove_var(cpt.to_bytes(name))
2445 2446
        del self.vars[name]

Y
Yu Yang 已提交
2447 2448
    def create_parameter(self, *args, **kwargs):
        global_block = self.program.global_block()
2449
        param = None
L
Leo Chen 已提交
2450
        if in_dygraph_mode():
2451
            param = ParamBase(*args, **kwargs)
L
Leo Chen 已提交
2452 2453
        else:
            param = Parameter(global_block, *args, **kwargs)
2454
        if 'initializer' in kwargs:
2455 2456 2457 2458 2459

            def _is_inited_by(block, var):
                init_ops = []
                for op in block.ops:
                    if var.name in op.output_arg_names:
2460 2461 2462 2463 2464
                        # In startup_program, "c_broadcast" and "c_sync_comm_stream"
                        # are treated as initialization ops that cause error. 
                        # Think of "c_broadcast" and "c_sync_comm_stream" as a special case here.
                        if op.type in ["c_broadcast", "c_sync_comm_stream"]:
                            continue
2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475
                        init_ops.append(op)
                return init_ops

            initializer = kwargs['initializer']
            init_ops = _is_inited_by(global_block, param)
            init_ops_len = len(init_ops)
            if init_ops_len > 1:
                raise RuntimeError("param " + param.name +
                                   " is inited by multiple init ops " + str(
                                       init_ops))
            elif init_ops_len == 1:
2476
                # TODO already inited, do nothing, should log a warning
2477 2478 2479
                pass
            else:
                initializer(param, self)
2480
        param.stop_gradient = False
Q
Qiao Longfei 已提交
2481
        return param
Y
Yu Yang 已提交
2482

Y
Yu Yang 已提交
2483
    def append_op(self, *args, **kwargs):
2484 2485 2486 2487 2488 2489
        """
        Appends a new Operator according to the giving arguments.

        Returns:
            Operator: the append Operator.
        """
L
lujun 已提交
2490
        if in_dygraph_mode():
2491 2492 2493
            attrs = kwargs.get("attrs", {})
            if _dygraph_tracer_._train_mode == False:
                # eval mode
2494 2495 2496 2497 2498
                if ('trainable_statistics' not in attrs
                    ) or not attrs['trainable_statistics']:
                    attrs['is_test'] = True
                else:
                    attrs['is_test'] = False
2499

J
Jiabin Yang 已提交
2500 2501
            type = kwargs.get("type", None)

2502 2503 2504
            op = Operator(
                block=self,
                desc=None,
J
Jiabin Yang 已提交
2505
                type=type,
M
minqiyang 已提交
2506 2507
                inputs=None,
                outputs=None,
2508
                attrs=attrs)
2509

M
minqiyang 已提交
2510 2511 2512
            # record ops in tracer rather than blocks
            #
            # TODO(minqiyang): add op stop_gradient support in static mode too.
L
lujun 已提交
2513
            # currently, we only support stop_gradient in dygraph mode.
J
Jiabin Yang 已提交
2514 2515

            _dygraph_tracer().trace_op(type,
M
minqiyang 已提交
2516
                                       kwargs.get("inputs", {}),
J
Jiabin Yang 已提交
2517 2518
                                       kwargs.get("outputs", {}), attrs
                                       if attrs else {},
M
minqiyang 已提交
2519
                                       kwargs.get("stop_gradient", False))
M
minqiyang 已提交
2520
        else:
2521 2522 2523 2524 2525 2526 2527 2528 2529
            op_desc = self.desc.append_op()
            op = Operator(
                block=self,
                desc=op_desc,
                type=kwargs.get("type", None),
                inputs=kwargs.get("inputs", None),
                outputs=kwargs.get("outputs", None),
                attrs=kwargs.get("attrs", None))

M
minqiyang 已提交
2530
            self.ops.append(op)
M
minqiyang 已提交
2531

2532 2533
        return op

W
Wu Yi 已提交
2534
    def _insert_op(self, index, *args, **kwargs):
2535 2536 2537 2538 2539 2540 2541 2542 2543
        """
        Insert a Operator according to the giving arguments.

        Args:
            index(int): the place that the operator to insert.

        Returns:
            Operator: the insert Operator.
        """
W
Wu Yi 已提交
2544 2545
        self._sync_with_cpp()
        op_desc = self.desc._insert_op(index)
Q
qiaolongfei 已提交
2546 2547 2548 2549
        op = Operator(block=self, desc=op_desc, *args, **kwargs)
        self.ops.insert(index, op)
        return op

W
Wu Yi 已提交
2550
    def _remove_op(self, index):
2551 2552 2553 2554 2555 2556 2557 2558 2559
        """
        Remove the specific position operator.

        Args:
            index(int): the position that the operator to insert.

        Returns:
            None
        """
W
Wu Yi 已提交
2560 2561
        self._sync_with_cpp()
        self.desc._remove_op(index, index + 1)
2562 2563
        del self.ops[index]

W
Wu Yi 已提交
2564
    def _slice_ops(self, start, end):
2565 2566 2567 2568 2569 2570 2571 2572 2573 2574
        """
        Return the Operator between start and end.

        Args:
            start(int): the start position.
            end(int): the end position.

        Returns:
            list: the Operators between start and end.
        """
Q
qiaolongfei 已提交
2575
        return self.ops[start:end]
Y
Yancey1989 已提交
2576

W
Wu Yi 已提交
2577
    def _prepend_op(self, *args, **kwargs):
L
lujun 已提交
2578
        if in_dygraph_mode():
J
Jiabin Yang 已提交
2579 2580
            type = kwargs.get("type", None)
            attrs = kwargs.get("attrs", {})
2581
            op = Operator(
J
Jiabin Yang 已提交
2582
                self, None, type=type, inputs=None, outputs=None, attrs=attrs)
M
minqiyang 已提交
2583

J
Jiabin Yang 已提交
2584
            _dygraph_tracer().trace_op(type,
M
minqiyang 已提交
2585
                                       kwargs.get("inputs", {}),
J
Jiabin Yang 已提交
2586 2587
                                       kwargs.get("outputs", {}), attrs
                                       if attrs else {},
M
minqiyang 已提交
2588
                                       kwargs.get("stop_gradient", False))
M
minqiyang 已提交
2589
        else:
2590 2591 2592 2593 2594 2595 2596 2597
            op_desc = self.desc._prepend_op()
            op = Operator(
                self,
                op_desc,
                type=kwargs.get("type", None),
                inputs=kwargs.get("inputs", None),
                outputs=kwargs.get("outputs", None),
                attrs=kwargs.get("attrs", None))
M
minqiyang 已提交
2598
            self.ops.insert(0, op)
2599

Y
Yu Yang 已提交
2600 2601
        return op

W
Wu Yi 已提交
2602
    def _sync_with_cpp(self):
2603
        """
2604 2605
        Sync from the desc on the c++ end. This method is used to synchronize
        the c++ desc instance generated by backward.
2606
        """
Q
Qiao Longfei 已提交
2607 2608 2609 2610 2611
        # sync variables from cpp
        for var in self.desc.all_vars():
            if not self.has_var(var.name()):
                self.create_var(name=var.name(), desc=var, type=var.type())

2612
        # sync variables removed from c++ end
2613
        for var in list(self.vars.keys()):
M
minqiyang 已提交
2614
            if not self.desc.find_var(cpt.to_bytes(var)):
2615 2616
                self.vars.pop(var)

Q
Qiao Longfei 已提交
2617
        # sync operators from cpp
2618 2619 2620 2621
        ops_in_cpp = []
        for op_idx in range(0, self.desc.op_size()):
            ops_in_cpp.append(self.desc.op(op_idx))

Y
Yu Yang 已提交
2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637
        if len(self.ops) != 0:
            first_op_in_python = self.ops[0].desc
            last_op_in_python = self.ops[len(self.ops) - 1].desc
            start_index = None
            end_index = None
            for index in range(len(ops_in_cpp)):
                if first_op_in_python == ops_in_cpp[index]:
                    start_index = index
                if last_op_in_python == ops_in_cpp[index]:
                    end_index = index
            assert start_index is not None
            assert end_index is not None
            assert start_index <= end_index
        else:
            start_index = 0
            end_index = -1
Q
Qiao Longfei 已提交
2638 2639 2640 2641 2642

        # sync ops append to the head of cpp_ops
        for index in range((start_index - 1 - 1), -1, -1):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
Q
qiaolongfei 已提交
2643
            self.ops.insert(0, op)
Q
Qiao Longfei 已提交
2644 2645 2646 2647 2648 2649 2650

        # sync ops append to the end of cpp_ops
        for index in range((end_index + 1), len(ops_in_cpp)):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
            self.ops.append(op)

2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663
        # sync ops removed from c++ end
        if end_index != -1 and end_index < len(self.ops):
            ops_in_cpp_index = 0
            ops_in_python_index = 0
            while ops_in_python_index < len(
                    self.ops) and ops_in_cpp_index < len(ops_in_cpp):
                if self.ops[ops_in_python_index].desc != ops_in_cpp[
                        ops_in_cpp_index]:
                    del self.ops[ops_in_python_index]
                else:
                    ops_in_cpp_index += 1
                    ops_in_python_index += 1

Q
Qiao Longfei 已提交
2664 2665 2666 2667
        assert len(self.ops) == len(ops_in_cpp)
        for index in range(len(self.ops)):
            assert self.ops[index].desc == ops_in_cpp[index]

W
Wu Yi 已提交
2668
    def _copy_param_info_from(self, other):
2669
        """
2670 2671
        Copy the information of parameters from the other block.

2672
        Args:
2673 2674 2675 2676 2677
            other(Block): the other block.

        Raises:
            ValueError: If type of input is not Block, or the `other` and this
                block is not in the same topology.
2678 2679 2680 2681 2682

        Returns:
            None
        """
        if not isinstance(other, Block):
W
Wu Yi 已提交
2683 2684
            raise TypeError(
                "_copy_param_info_from should be invoked with Block")
2685
        for p in other.iter_parameters():
2686 2687 2688
            assert isinstance(p, Parameter)
            v = self.vars.get(p.name, None)
            if v is None:
W
Wu Yi 已提交
2689
                raise ValueError("_copy_param_info_from should be invoked with "
2690 2691
                                 "same topology")
            assert isinstance(v, Variable)
2692
            new_p = None
L
Leo Chen 已提交
2693 2694
            if in_dygraph_mode():
                new_p = ParamBase(
2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706
                    shape=v.shape,
                    dtype=v.dtype,
                    type=v.type,
                    lod_level=v.lod_level,
                    stop_gradient=p.stop_gradient,
                    trainable=p.trainable,
                    optimize_attr=p.optimize_attr,
                    regularizer=p.regularizer,
                    gradient_clip_attr=p.gradient_clip_attr,
                    error_clip=p.error_clip,
                    name=v.name)
            else:
L
Leo Chen 已提交
2707 2708
                new_p = Parameter(
                    block=self,
2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719
                    shape=v.shape,
                    dtype=v.dtype,
                    type=v.type,
                    lod_level=v.lod_level,
                    stop_gradient=p.stop_gradient,
                    trainable=p.trainable,
                    optimize_attr=p.optimize_attr,
                    regularizer=p.regularizer,
                    gradient_clip_attr=p.gradient_clip_attr,
                    error_clip=p.error_clip,
                    name=v.name)
2720 2721
            self.vars[new_p.name] = new_p

2722
    def _clone_variable(self, var, force_persistable=True):
2723 2724
        """
        Clone a variable into current block.
2725

2726 2727
        Args:
            var: the variable to be cloned.
2728 2729 2730
            force_persistable(bool): True means setting the result variable to being persistable.
                                     False means setting the persistable the same with that of input var.
                                     default: True.
2731 2732

        Returns:
2733
            Variable: the new  variable cloned from 'var' in current block.
2734 2735
        """
        assert isinstance(var, Variable)
T
update  
typhoonzero 已提交
2736 2737 2738 2739 2740
        ret_var = None
        # make STEP_SCOPES var can be safely cloned.
        if var.type == core.VarDesc.VarType.STEP_SCOPES:
            ret_var = self.create_var(
                name=var.name, persistable=var.persistable, type=var.type)
T
tangwei12 已提交
2741 2742
        elif var.type == core.VarDesc.VarType.RAW:
            ret_var = self.create_var(
T
tangwei12 已提交
2743
                name=var.name, persistable=var.persistable, type=var.type)
T
typhoonzero 已提交
2744 2745 2746 2747 2748 2749
        elif var.type == core.VarDesc.VarType.SELECTED_ROWS:
            ret_var = self.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
2750
                persistable=True if force_persistable else var.persistable,
H
Huihuang Zheng 已提交
2751 2752
                is_data=var.is_data,
                need_check_feed=var.desc.need_check_feed())
T
update  
typhoonzero 已提交
2753 2754 2755 2756 2757 2758 2759
        else:
            ret_var = self.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
                lod_level=var.lod_level,
2760
                persistable=True if force_persistable else var.persistable,
H
Huihuang Zheng 已提交
2761 2762
                is_data=var.is_data,
                need_check_feed=var.desc.need_check_feed())
T
update  
typhoonzero 已提交
2763
        return ret_var
2764

Y
Yu Yang 已提交
2765

2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860
class IrNode(object):
    """
    Python IrNode. Beneath it is a core.Node, which is used for Ir Pass.
    """

    def __init__(self, node):
        """
        Construct an IrNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node,
                          core.Node), 'node must be the instance of core.Node.'
        self.node = node

    def name(self):
        """
        Return the node name.

        Returns:
            str: node name.
        """
        return self.node.name()

    def node_type(self):
        """
        Return the node type.

        Returns:
            core.Node.Type: node type(core.Node.Type.Operation or core.Node.Type.Variable).
        """
        return self.node.node_type()

    def var(self):
        """
        Return the node variable description.

        Returns:
            core.VarDesc: node variable description.
        """
        return self.node.var()

    def op(self):
        """
        Return the node operator description.

        Returns:
            core.OpDesc: node operator description.
        """
        return self.node.op()

    def id(self):
        """
        Return the node id.

        Returns:
            int: node id.
        """
        return self.node.id()

    def is_op(self):
        """
        If the node is an operator, then return true.

        Returns:
            bool: indicate whether the node is an operator.
        """
        return self.node.is_op()

    def is_var(self):
        """
        If the node is a variable, then return true.

        Returns:
            bool: indicate whether the node is a variable.
        """
        return self.node.is_var()

    def is_ctrl_var(self):
        """
        If the node is a control dependence variable, then return true.

        Returns:
            bool: indicate whether the node is a control dependence variable.
        """
        return self.node.is_ctrl_var()

    def clear_inputs(self):
        """
        Clear the node inputs. After executing the `clear_inputs` function,
        the node inputs will be empty.
        """
        self.node.clear_inputs()

2861
    def remove_input_by_id(self, node_id):
2862 2863 2864 2865 2866 2867
        """
        Remove a node from inputs by the given node id.

        Args:
            node_id(int): the given node id.
        """
2868
        self.node.remove_input(node_id)
2869

2870
    def remove_input(self, node):
2871 2872 2873 2874
        """
        Remove a node from inputs.

        Args:
2875
            node(IrNode): the node being removed.
2876
        """
2877
        self.node.remove_input(node.node)
2878

2879
    def append_input(self, node):
2880 2881 2882 2883
        """
        Append a node in inputs.

        Args:
2884
            node(IrNode): the node being appended.
2885
        """
2886
        self.node.append_input(node.node)
2887 2888 2889 2890 2891 2892 2893 2894

    def clear_outputs(self):
        """
        Clear the node outputs. After executing the `clear_outputs` function,
        the node outputs will be empty.
        """
        self.node.clear_outputs()

2895
    def remove_output_by_id(self, node_id):
2896 2897 2898 2899 2900 2901
        """
        Remove a node from outputs by the given node id.

        Args:
            node_id(int): the given node id.
        """
2902
        self.node.remove_output(node_id)
2903

2904
    def remove_output(self, node):
2905 2906 2907 2908
        """
        Remove a node from outputs.

        Args:
2909
            node(IrNode): the node being removed.
2910
        """
2911
        self.node.remove_output(node.node)
2912

2913
    def append_output(self, node):
2914 2915 2916 2917
        """
        Append a node in outputs.

        Args:
2918
            node(IrNode): the node being appended.
2919
        """
2920
        self.node.append_output(node.node)
2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967

    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrNode): node inputs wrapped by IrNode.
        """
        return [IrNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrNode): node outputs wrapped by IrNode.
        """
        return [IrNode(n) for n in self.node.outputs]


class IrVarNode(IrNode):
    """
    Python IrVarNode. Beneath it is a core.Node, it inherits from IrNode.
    """

    def __init__(self, node):
        """
        Construct an IrVarNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node, core.Node) and node.is_var(), \
            'node must be the instance of core.Node and it must be a variable node.'
        super(IrVarNode, self).__init__(node)
        self.node = node

    def set_shape(self, shape):
        """
        Set the node variable shape.

        Args:
            shape(list): shape to be set.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
2968
            "The node variable description can not be None."
2969 2970 2971 2972 2973 2974 2975 2976 2977 2978
        self.node.var().set_shape(shape)

    def persistable(self):
        """
        If the variable node is a persistable variable, then return true.

        Returns:
            bool: indicate whether the variable is persistable.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
2979
            "The node variable description can not be None."
2980 2981
        return self.node.var().persistable()

2982 2983 2984 2985 2986 2987 2988 2989
    def type(self):
        """
        Return the variable type.

        Returns:
            core.VarDesc.VarType: the variable type.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
2990
            "The node variable description can not be None."
2991 2992 2993 2994 2995 2996 2997 2998 2999 3000
        return self.node.var().type()

    def dtype(self):
        """
        Return the variable data type.

        Returns:
            core.VarDesc.VarType: the variable data type.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
3001
            "The node variable description can not be None."
3002 3003 3004 3005 3006 3007 3008 3009 3010 3011
        return self.node.var().dtype()

    def shape(self):
        """
        Return the variable shape.

        Returns:
            list: the variable shape.
        """
        assert self.node.var() is not None, \
T
tianshuo78520a 已提交
3012
            "The node variable description can not be None."
3013 3014
        return self.node.var().shape()

3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061
    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrOpNode): node inputs wrapped by IrOpNode.
        """
        return [IrOpNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrOpNode): node outputs wrapped by IrOpNode.
        """
        return [IrOpNode(n) for n in self.node.outputs]


class IrOpNode(IrNode):
    """
    Python IrOpNode. Beneath it is a core.Node, it inherits from IrNode.
    """

    def __init__(self, node):
        """
        Construct an IrOpNode using core.Node.

        Args:
            node(core.Node): C++ Node.
        """
        assert isinstance(node, core.Node) and node.is_op(), \
            'node must be the instance of core.Node and it must be a operator node.'
        super(IrOpNode, self).__init__(node)
        self.node = node

    def rename_input(self, old_input_name, new_input_name):
        """
        Rename the input of this node.

        Args:
            old_input_name(str): the old input name.
            new_input_name(str): the new input name.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
3062
            "The node operator description can not be None."
3063 3064
        self.node.op()._rename_input(old_input_name, new_input_name)

3065 3066 3067 3068 3069 3070 3071 3072 3073
    def rename_output(self, old_output_name, new_output_name):
        """
        Rename the output of this node.

        Args:
            old_output_name(str): the old output name.
            new_output_name(str): the new output name.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
3074
            "The node operator description can not be None."
3075 3076 3077 3078
        print("op: {}, old: {}, new: {}\n".format(self.node.op().type(
        ), old_output_name, new_output_name))
        self.node.op()._rename_output(old_output_name, new_output_name)

3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089
    def input(self, name):
        """
        Get the argument name list by the parameter name for input.

        Args:
            name(str): the parameter name.

        Returns:
            list(str): the argument name list.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
3090
            "The node operator description can not be None."
3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103
        return self.node.op().input(name)

    def output(self, name):
        """
        Get the argument name list by the parameter name for output.

        Args:
            name(str): the parameter name.

        Returns:
            list(str): the argument name list.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
3104
            "The node operator description can not be None."
3105 3106 3107 3108 3109 3110 3111 3112 3113 3114
        return self.node.op().output(name)

    def set_type(self, new_type):
        """
        Change the operator type into new type.

        Args:
            new_type(str): new operator type to be set.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
3115
            "The node operator description can not be None."
3116 3117
        return self.node.op().set_type(new_type)

3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132
    def set_attr(self, name, val):
        """
        Set the value of attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.
        """
        self._update_desc_attr(name, val)

    def _update_desc_attr(self, name, val):
        """
        Update the value of the op desc's attribute by attribute's name.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
3133
            "The node operator description can not be None."
3134 3135 3136 3137
        desc = self.node.op()
        if isinstance(val, Block):
            desc.set_block_attr(name, val.desc)
        elif isinstance(val, list) and val and \
3138
                all(isinstance(v, Block) for v in val):
3139 3140
            desc.set_blocks_attr(name, [v.desc for v in val])
        elif isinstance(val, core.BlockDesc) or \
3141
                isinstance(val, core.ProgramDesc):
3142 3143 3144 3145
            desc.set_serialized_attr(name, val.serialize_to_string())
        else:
            desc._set_attr(name, val)

3146 3147 3148 3149 3150 3151 3152 3153
    def input_arg_names(self):
        """
        Return input arguments' names of this op node.

        Returns:
            list(str): input arguments' names of this op node.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
3154
            "The node operator description can not be None."
3155 3156 3157 3158 3159 3160 3161 3162 3163 3164
        return self.node.op().input_arg_names()

    def output_arg_names(self):
        """
        Return output arguments' names of this op node.

        Returns:
            list(str): output arguments' names of this op node.
        """
        assert self.node.op() is not None, \
T
tianshuo78520a 已提交
3165
            "The node operator description can not be None."
3166 3167
        return self.node.op().output_arg_names()

3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188
    @property
    def inputs(self):
        """
        Return the node inputs.

        Returns:
            list(IrVarNode): node inputs wrapped by IrVarNode.
        """
        return [IrVarNode(n) for n in self.node.inputs]

    @property
    def outputs(self):
        """
        Return the node outputs.

        Returns:
            list(IrVarNode): node outputs wrapped by IrVarNode.
        """
        return [IrVarNode(n) for n in self.node.outputs]


3189 3190
class IrGraph(object):
    """
3191
    Python IrGraph. Beneath it is a core.Graph, which is used for
3192
    creating a c++ Ir Pass Graph. An IrGraph is just a graph view of
3193 3194
    a Program. In an IrGraph, both Variables and Operators are graph
    nodes.
3195 3196 3197 3198
    """

    def __init__(self, graph, for_test=False):
        """
3199 3200
        Construct an IrGraph using core.Graph.

3201 3202 3203 3204 3205 3206 3207 3208 3209
        Args:
            graph(core.Graph): C++ Graph.
            for_test(bool): True for the test graph and false for the train graph.
        """
        assert isinstance(
            graph, core.Graph), 'graph must be the instance of core.Graph.'
        self.graph = graph
        self._for_test = for_test

3210 3211 3212 3213
    def clone(self):
        """
        Create a new and duplicated IrGraph.

3214 3215 3216
        Warns:
            The method only clones the graph structure, not its attributes.

3217 3218 3219
        Returns:
            IrGraph: A new and duplicated graph.
        """
3220
        g = self.graph.clone()
3221 3222
        return IrGraph(g, self._for_test)

3223
    def is_test(self):
3224 3225 3226
        """
        If the graph is used for testing, the function returns true. Otherwise, returns false.
        """
3227 3228
        return self._for_test

W
WangZhen 已提交
3229
    def all_nodes(self):
3230 3231 3232
        """
        Return all nodes included in the graph as a set.
        """
3233
        return {IrNode(node) for node in self.graph.nodes()}
3234

3235
    def all_var_nodes(self):
3236 3237 3238
        """
        Return all variable nodes included in the graph as a set.
        """
3239
        return {IrVarNode(node) for node in self.graph.nodes() if node.is_var()}
3240

3241
    def all_persistable_nodes(self):
3242 3243 3244
        """
        Return all persistable variable nodes included in the graph as a set.
        """
W
WangZhen 已提交
3245 3246 3247 3248 3249
        persistable_nodes = set()
        for node in self.graph.nodes():
            if node.is_var() and node.var() is not None and node.var(
            ).persistable():
                persistable_nodes.add(node)
3250
        return {IrVarNode(p) for p in persistable_nodes}
W
WangZhen 已提交
3251

3252
    def all_op_nodes(self):
3253 3254 3255
        """
        Return all operator nodes included in the graph as a set.
        """
3256
        return {IrOpNode(node) for node in self.graph.nodes() if node.is_op()}
3257

3258
    def create_persistable_node(self, name, var_type, shape, var_dtype):
3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269
        """
        Create a persistable variable node in the graph. In IrGraph,
        it can not distinguish between persistable variables and parameters.

        Args:
            name(str): the name of the persistable variable node.
            vart_type(core.VarDesc.VarType): the type of the persistable variable node.
            shape(list): the shape of the persistable variable node.
            var_dtype(core.VarDesc.VarType): the data type of the persistable variable node.

        Returns:
3270
            IrVarNode: the created persistable variable node.
3271
        """
3272 3273 3274 3275 3276
        var_desc = core.VarDesc(name)
        var_desc.set_type(var_type)
        var_desc.set_shape(shape)
        var_desc.set_dtype(var_dtype)
        var_desc.set_persistable(True)
3277
        return IrVarNode(self.graph.create_var_node(var_desc))
3278 3279

    def create_var_node(self, name, var_type, shape, var_dtype):
3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290
        """
        Create a variable node in the graph. The created variable node is
        not persistable.

        Args:
            name(str): the name of the variable node.
            vart_type(core.VarDesc.VarType): the type of the variable node.
            shape(list): the shape of the variable node.
            var_dtype(core.VarDesc.VarType): the data type of the variable node.

        Returns:
3291
            IrVarNode: the created variable node.
3292 3293
        """

3294 3295 3296 3297
        var_desc = core.VarDesc(name)
        var_desc.set_type(var_type)
        var_desc.set_shape(shape)
        var_desc.set_dtype(var_dtype)
3298
        return IrVarNode(self.graph.create_var_node(var_desc))
3299 3300

    def create_var_node_from_desc(self, var_desc):
3301 3302 3303 3304 3305 3306 3307 3308
        """
        Create a variable node by using an existing VarDesc in the graph.
        Depend on the giving VarDesc, the created variable node may be persistable.

        Args:
            var_desc(core.VarDesc): the giving variable description.

        Returns:
3309
            IrVarNode: the created variable node.
3310
        """
3311
        return IrVarNode(self.graph.create_var_node(var_desc))
3312 3313

    def create_op_node(self, op_type, attrs, inputs, outputs):
3314 3315 3316 3317 3318 3319 3320
        """
        Create a operator node in the graph.

        Args:
            op_type(str): the type of the operator node.
            attrs(dict): the attributes of the operator node.
            inputs(dict): the inputs of the operator node.
T
tianshuo78520a 已提交
3321
            outputs(dict): the outputs of the operator node.
3322 3323

        Returns:
3324
            IrOpNode: the created operator node.
3325
        """
3326 3327
        op_desc = core.OpDesc()
        op_desc.set_type(op_type)
3328
        for attr, value in six.iteritems(attrs):
3329
            self._update_desc_attr(op_desc, attr, value)
3330
        for input_name, var_nodes in six.iteritems(inputs):
3331 3332 3333 3334
            if not isinstance(var_nodes, list):
                var_nodes = [var_nodes]
            op_desc.set_input(input_name,
                              [var_node.name() for var_node in var_nodes])
3335
        for output_name, var_nodes in six.iteritems(outputs):
3336 3337 3338 3339
            if not isinstance(var_nodes, list):
                var_nodes = [var_nodes]
            op_desc.set_output(output_name,
                               [var_node.name() for var_node in var_nodes])
3340
        return IrOpNode(self.graph.create_op_node(op_desc))
3341 3342

    def create_op_node_from_desc(self, op_desc):
3343 3344 3345 3346 3347 3348 3349
        """
        Create a operator node by using an existing OpDesc in the graph.

        Args:
            op_desc(core.VarDesc): the giving operator description.

        Returns:
3350
            IrOpNode: the created operator node.
3351
        """
3352
        return IrOpNode(self.graph.create_op_node(op_desc))
3353 3354

    def update_input_link(self, old_input_node, new_input_node, op_node):
3355 3356 3357 3358
        """
        Update the input's link of a operator node.

        Args:
3359 3360 3361
            old_input_node(IrNode): the old input node of the giving op_node.
            new_input_node(IrNode): the new input node of the giving op_node.
            op_node(IrOpNode): the operator node that is needed to update input's link.
3362
        """
3363
        assert old_input_node.node in self.graph.nodes() and new_input_node.node in \
3364 3365
               self.graph.nodes() and op_node.node in self.graph.nodes(), \
            'The three arguments(old_input_node&new_input_node&op_node) must be in the graph nodes.'
3366 3367 3368 3369
        old_input_node.remove_output(op_node)
        op_node.remove_input(old_input_node)
        new_input_node.append_output(op_node)
        op_node.append_input(new_input_node)
3370
        op_node.rename_input(old_input_node.name(), new_input_node.name())
3371

3372 3373 3374 3375 3376 3377 3378 3379 3380 3381
    def update_output_link(self, old_output_node, new_output_node, op_node):
        """
        Update the output's link of an operator node.

        Args:
            old_output_node(IrNode): the old output node of the giving op_node.
            new_output_node(IrNode): the new output node of the giving op_node.
            op_node(IrOpNode): the operator node that is needed to update input's link.
        """
        assert old_output_node.node in self.graph.nodes() and new_output_node.node in \
3382 3383
               self.graph.nodes() and op_node.node in self.graph.nodes(), \
            'The three arguments(old_output_node &new_output_node &op_node) must be in the graph nodes.'
3384 3385 3386 3387 3388 3389
        old_output_node.remove_input(op_node)
        op_node.remove_output(old_output_node)
        new_output_node.append_input(op_node)
        op_node.append_output(new_output_node)
        op_node.rename_output(old_output_node.name(), new_output_node.name())

3390
    def link_to(self, node_in, node_out):
3391 3392 3393 3394
        """
        Connect two nodes.

        Args:
3395 3396
            node_in(IrNode): the input node.
            node_out(IrNode): the output node.
3397
        """
3398
        assert node_in.node in self.graph.nodes() and node_out.node in self.graph.nodes(), \
W
WangZhen 已提交
3399
            'The two arguments(node_in&node_out) must be in the graph nodes.'
3400 3401
        node_in.append_output(node_out)
        node_out.append_input(node_in)
3402 3403

    def safe_remove_nodes(self, remove_nodes):
3404 3405 3406 3407 3408 3409 3410
        """
        Remove nodes safely since links connected to these removed nodes are
        also removed.

        Args:
            remove_nodes(set): the nodes prepared to be removed.
        """
3411
        if not isinstance(remove_nodes, set):
W
WangZhen 已提交
3412 3413 3414 3415
            if isinstance(remove_nodes, Iterable):
                remove_nodes = set(remove_nodes)
            else:
                remove_nodes = {remove_nodes}
3416 3417
        original_nodes = {n.node for n in remove_nodes}
        core.graph_safe_remove_nodes(self.graph, original_nodes)
3418

Z
Zhen Wang 已提交
3419 3420 3421 3422 3423 3424 3425 3426
    def resolve_hazard(self):
        ordered_nodes = core.topology_sort(self.graph)
        var_nodes = dict()
        for node in ordered_nodes:
            if node.is_op() and node.op() is not None:
                for each_var_name in node.op().input_arg_names():
                    if each_var_name not in var_nodes:
                        var_nodes[each_var_name] = [
3427
                            self._find_node_by_name(node.inputs, each_var_name)
Z
Zhen Wang 已提交
3428 3429 3430 3431
                        ]
                for each_var_name in node.op().output_arg_names():
                    if each_var_name not in var_nodes:
                        var_nodes[each_var_name] = [
3432
                            self._find_node_by_name(node.outputs, each_var_name)
Z
Zhen Wang 已提交
3433 3434 3435
                        ]
                    else:
                        var_nodes[each_var_name].append(
3436 3437
                            self._find_node_by_name(node.outputs,
                                                    each_var_name))
Z
Zhen Wang 已提交
3438 3439
        self.graph.resolve_hazard(var_nodes)

W
WangZhen 已提交
3440
    def has_circle(self):
3441 3442 3443 3444 3445 3446
        """
        Check if the graph has a circle.

        Returns:
            bool: True if the graph has a circle else False.
        """
W
WangZhen 已提交
3447 3448 3449
        return core.has_circle(self.graph)

    def graph_num(self):
3450 3451 3452 3453 3454 3455
        """
        Count the number of unconnected graphs in this graph.

        Returns:
            int: the number of unconnected graphs.
        """
W
WangZhen 已提交
3456 3457 3458
        return core.graph_num(self.graph)

    def topology_sort(self):
3459 3460 3461
        """
        Perform the topology sort operation on the graph.

T
tianshuo78520a 已提交
3462
        Notes: the `graph` can not contain a circle.
3463 3464

        Returns:
Z
Zhen Wang 已提交
3465
            list(IrNode): nodes in topology order.
3466
        """
3467
        ordered_nodes = core.topology_sort(self.graph)
Z
Zhen Wang 已提交
3468
        return [IrNode(n) for n in ordered_nodes]
W
WangZhen 已提交
3469 3470

    def build_adjacency_list(self):
3471 3472 3473 3474
        """
        Build an adjacency list of operations for the `graph`.

        Returns:
3475
            dict{IrNode: set(IrNode)}: the adjacency list.
3476
        """
3477 3478 3479 3480 3481
        adj_list = core.build_adjacency_list(self.graph)
        wrapped_adj_list = dict()
        for k, v in six.iteritems(adj_list):
            wrapped_adj_list[IrNode(k)] = {IrNode(n) for n in v}
        return wrapped_adj_list
W
WangZhen 已提交
3482

3483 3484 3485 3486 3487 3488 3489 3490
    def draw(self, save_path, name, marked_nodes=None, remove_ctr_var=True):
        """
        Draw the graph. If `dot` command is installed, the drawn graph
        will be saved as pdf file type, otherwise dot file type is used.

        Args:
            save_path(str): the save path of drawn graph.
            name(str): the name of drawn graph.
3491
            marked_nodes(set(IrNode)): nodes that are needed to be marked.
3492 3493 3494 3495 3496
            Default value is None.
            remove_ctr_var(bool): If it is set True, all control variable nodes
            in the graph will be removed. Default value is True.
        """

3497 3498 3499
        def _convert_to_pdf(dot_file_path):
            pdf_save_path = os.path.splitext(dot_file_path)[0] + '.pdf'
            exited_code = subprocess.call('dot -Tpdf ' + dot_file_path \
3500
                                          + ' -o ' + pdf_save_path, shell=True)
3501 3502 3503 3504 3505
            if exited_code != 0:
                print('The dot command is needed for creating pdf files.')
                print('The {} is saved as the dot filetype.'.format(
                    dot_file_path))

3506
        remove_ctr_vars = set()
3507
        if remove_ctr_var:
3508
            for node in self.all_var_nodes():
3509 3510 3511
                if node.is_ctrl_var():
                    remove_ctr_vars.add(node)
            self.safe_remove_nodes(remove_ctr_vars)
3512 3513
        print('Total ops num = {}.'.format(len(self.all_op_nodes())))

3514 3515
        if marked_nodes is not None:
            if not isinstance(marked_nodes, set):
3516 3517 3518 3519 3520 3521
                if isinstance(marked_nodes, Iterable):
                    marked_nodes = set(marked_nodes)
                else:
                    marked_nodes = {marked_nodes}
            marked_nodes = {n.node for n in marked_nodes}
            remove_ctr_vars = {n.node for n in remove_ctr_vars}
3522 3523 3524 3525
            marked_nodes = marked_nodes - remove_ctr_vars
            if self.graph.has('__graphviz__marked_node__'):
                self.graph.erase('__graphviz__marked_node__')
            self.graph.set('__graphviz__marked_node__', marked_nodes)
3526 3527
        if not os.path.exists(save_path):
            os.makedirs(save_path)
3528 3529 3530 3531 3532 3533 3534
        viz_dot_path = os.path.join(save_path, name) + '.dot'
        viz_pass = core.get_pass('graph_viz_pass')
        viz_pass.set('graph_viz_path', viz_dot_path)
        viz_pass.apply(self.graph)
        _convert_to_pdf(viz_dot_path)

    def to_program(self):
3535 3536 3537
        """
        Convert the graph into a Program.

Z
Zhen Wang 已提交
3538
        WARN: When the graph includes backward operator nodes, the
3539 3540 3541 3542 3543 3544
        conversion process may be failed. Usually, this function is
        only used to convert a test graph.

        Returns:
            Program: a program converted from the graph.
        """
3545
        convert_pass = core.get_pass('graph_to_program_pass')
3546 3547
        desc = core.ProgramDesc()
        convert_pass.set_not_owned('program', desc)
3548 3549 3550 3551
        convert_pass.apply(self.graph)
        program = Program._construct_from_desc(desc)
        return program

3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562
    def _find_node_by_name(self, nodes, node_name):
        """
        Find a node in the giving nodes set by the name.
        """
        target_node = None
        for n in nodes:
            if n.name() == node_name:
                target_node = n
        assert target_node is not None, "Cannot find the target node in the giving set."
        return target_node

3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578
    def _update_desc_attr(self, desc, name, val):
        """
        Update the value of desc's attribute by attribute's name.
        """
        if isinstance(val, Block):
            desc.set_block_attr(name, val.desc)
        elif isinstance(val, list) and val and all(
                isinstance(v, Block) for v in val):
            desc.set_blocks_attr(name, [v.desc for v in val])
        elif isinstance(val, core.BlockDesc) or \
                isinstance(val, core.ProgramDesc):
            desc.set_serialized_attr(name, val.serialize_to_string())
        else:
            desc._set_attr(name, val)


Y
Yu Yang 已提交
3579
class Program(object):
D
dzhwinter 已提交
3580
    """
3581 3582
    Create Python Program.  It has at least one :ref:`api_guide_Block_en`, when the
    control flow op like conditional_block, while :ref:`api_fluid_layers_While` is included,
J
Jiabin Yang 已提交
3583
    it will contain nested block.
3584

J
Jiabin Yang 已提交
3585 3586 3587
    Please reference the
    `framework.proto <https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/framework/framework.proto>`_
    for details.
D
dzhwinter 已提交
3588

J
Jiabin Yang 已提交
3589
    A set of Program usually contains startup program and main program.
J
Jiabin Yang 已提交
3590
    A startup program is set to contain some initial work, eg. initialize the ``Parameter``, and the main
J
Jiabin Yang 已提交
3591 3592 3593 3594 3595 3596 3597
    program will contain the network structure and vars for train.

    A set of Program can be used for test or train, in train program ,
    Paddle will contain all content to build a train network,  in test
    program Paddle will prune some content which is irrelevant to test, eg.
    backward ops and vars.

J
Jiabin Yang 已提交
3598 3599 3600 3601
    **Notes**:
        **we have** :ref:`api_fluid_default_startup_program` **and** :ref:`api_fluid_default_main_program`
        **by default, a pair of them will shared the parameters. The** :ref:`api_fluid_default_startup_program` **only run once to initialize parameters,**
        :ref:`api_fluid_default_main_program` **run in every mini batch and adjust the weights.**
D
dzhwinter 已提交
3602 3603

    Returns:
J
Jiabin Yang 已提交
3604
        Program: An empty Program.
D
dzhwinter 已提交
3605 3606

    Examples:
3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619
        .. code-block:: python

            import paddle.fluid as fluid

            main_program = fluid.Program()
            startup_program = fluid.Program()
            with fluid.program_guard(main_program=main_program, startup_program=startup_program):
                x = fluid.layers.data(name="x", shape=[-1, 784], dtype='float32')
                y = fluid.layers.data(name="y", shape=[-1, 1], dtype='int32')
                z = fluid.layers.fc(name="fc", input=x, size=10, act="relu")

            print("main program is: {}".format(main_program))
            print("start up program is: {}".format(startup_program))
D
dzhwinter 已提交
3620 3621 3622

    """

3623 3624
    def __init__(self):
        self.desc = core.ProgramDesc()
Y
Yu Yang 已提交
3625 3626
        self.blocks = [Block(self, 0)]
        self.current_block_idx = 0
D
dzhwinter 已提交
3627
        self._seed = 0
Y
yuyang18 已提交
3628
        self._current_role = core.op_proto_and_checker_maker.OpRole.Forward
3629
        self.__op_role_var = []
T
tangwei12 已提交
3630

3631 3632
        # for distribute training
        # _is_distributed = True if under distributed training
T
tangwei12 已提交
3633
        self._is_distributed = False
3634
        # _is_chief = True if the trainer is the first one, usually No.0
T
tangwei12 已提交
3635
        self._is_chief = False
3636 3637 3638
        # _parameters_on_pservers records all the parameters distributed on parameter servers.
        self._parameters_on_pservers = None
        # _endpoints is a list about parameter servers ip:port, such as ["ip:port","ip:port"]
T
tangwei12 已提交
3639
        self._endpoints = []
3640 3641 3642
        # if current role is parameter server, the _ps_endpoint is its "ip:port"
        self._ps_endpoint = None
        # trainers_endpoints, it is used for distribution.
3643
        self._trainers_endpoints = []
3644
        # the distributed lookup table names
T
tangwei12 已提交
3645
        self._distributed_lookup_table = None
3646 3647 3648

        # use Deep gradient comrepssion or not
        self._enable_dgc = False
3649 3650
        self._use_lamb = False

3651 3652 3653
        self._nccl_comm_num = 1
        self._use_hierarchical_allreduce = False
        self._hierarchical_allreduce_inter_nranks = 0
3654

3655 3656 3657
        # if this program has been optimized by distributed optimizer
        # fleet_opt will be given a value
        self._fleet_opt = None
D
dongdaxiang 已提交
3658
        self._program_config = None
3659

H
hutuxian 已提交
3660 3661 3662
        # assigned if this program has been parsed by a pipeline optimizer
        self._pipeline_opt = None

3663 3664 3665
        # appending gradients times
        self._appending_grad_times = 0

Y
yuyang18 已提交
3666
    @property
3667
    def _op_role(self):
Y
yuyang18 已提交
3668 3669 3670 3671 3672 3673 3674 3675
        """
        The operator role. In a enum {Forward, Backward, Optimize}.

        Notes: this is a low level API. It is used only for ParallelExecutor to
        duplicate or schedule operator to devices.

        For example, the forward operator should be executed on every device.
        The backward operator should be executed on every device and the
3676
        parameter gradient of backward (use :code:`_op_role_var` to get this
Y
yuyang18 已提交
3677 3678 3679 3680
        variable) operator should be merged to one device. The optimization
        operators should be executed on only one device and broadcast the
        optimization result, i.e., the new parameter, to every other device.
        """
Y
yuyang18 已提交
3681 3682
        return self._current_role

3683 3684
    @_op_role.setter
    def _op_role(self, role):
Y
yuyang18 已提交
3685 3686 3687
        self._current_role = role

    @property
3688
    def _op_role_var(self):
Y
yuyang18 已提交
3689
        """
3690
        The auxiliary variables for :code:`_op_role` property.
Y
yuyang18 已提交
3691

3692
        See Also: :code:`Program._op_role`'s documentation for details.
Y
yuyang18 已提交
3693 3694 3695

        Notes: This is a very low-level API. Users should not use it directly.
        """
3696
        return self.__op_role_var
Y
yuyang18 已提交
3697

3698 3699 3700 3701 3702 3703 3704 3705 3706
    @contextlib.contextmanager
    def _backward_role_guard(self):
        tmp_role = self._current_role

        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.Backward
        yield
        self._current_role = tmp_role

S
rename  
sneaxiy 已提交
3707
    @signature_safe_contextmanager
W
Wu Yi 已提交
3708
    def _optimized_guard(self, param_and_grads):
Y
yuyang18 已提交
3709 3710 3711 3712 3713 3714 3715
        """
        A with guard to set :code:`Optimization` :code:`OpRole` and
        :code:`OpRoleVar` automatically.

        Notes: This is a very low level API. Users should not use it directly.

        Args:
3716
            param_and_grads(list): The variables (names) to be optimized.
Y
yuyang18 已提交
3717 3718 3719

        Examples:

3720
            >>> import paddle.fluid as fluid
Y
yuyang18 已提交
3721
            >>> p, g = backward(...)
W
Wu Yi 已提交
3722
            >>> with program._optimized_guard([p,g]):
Y
yuyang18 已提交
3723 3724
            >>>     p = p - 0.001 * g
        """
X
Xin Pan 已提交
3725
        tmp_role = self._current_role
3726
        tmp_var = self.__op_role_var
X
Xin Pan 已提交
3727

Y
yuyang18 已提交
3728 3729
        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.Optimize
3730
        self.__op_role_var = [
3731 3732 3733
            var.name if isinstance(var, Variable) else var
            for var in param_and_grads
        ]
Y
yuyang18 已提交
3734
        yield
3735
        self.__op_role_var = tmp_var
X
Xin Pan 已提交
3736
        self._current_role = tmp_role
Y
Yu Yang 已提交
3737

S
rename  
sneaxiy 已提交
3738
    @signature_safe_contextmanager
X
Xin Pan 已提交
3739
    def _lr_schedule_guard(self, is_with_opt=False):
3740 3741 3742 3743 3744 3745 3746
        """
        A with guard to set :code:`LRSched` :code:`OpRole` and
        :code:`OpRoleVar` automatically. The :code:`OpRoleVar` is
        set to the target learning rate.

        Notes: This is a very low level API. Users should not use it directly.

X
Xin Pan 已提交
3747 3748 3749 3750
        Args:
            is_with_opt: Only set to true if these ops a in the middle
                 of a bunch of optimize ops so that it can be treated
                 correctly. For example, sgd->lr_op->sgd->lr_op->sgd.
3751 3752 3753

        Examples:

3754
            >>> import paddle.fluid as fluid
3755 3756 3757 3758
            >>> p, g = backward(...)
            >>> with program.lr_schedule_guard():
            >>>     lr = lr * decay
        """
3759 3760

        tmp_role = self._current_role
3761
        tmp_var = self.__op_role_var
3762

3763 3764
        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.LRSched
X
Xin Pan 已提交
3765 3766
        if is_with_opt:
            self._current_role = int(OpRole.LRSched) | int(OpRole.Optimize)
3767
        # TODO(typhoonzero): how to set target learning rate var
3768
        self.__op_role_var = []
3769
        yield
3770
        self.__op_role_var = tmp_var
3771
        self._current_role = tmp_role
3772

3773
    def __str__(self):
Y
yuyang18 已提交
3774 3775 3776 3777 3778 3779 3780 3781 3782
        """
        Get the protobuf debug string of this Program.

        Returns:
            (str): The protobuf debug string.

        Raises:
            ValueError: If any of required fields is not set.
        """
Y
Yang Yang(Tony) 已提交
3783 3784
        return self.to_string(True)

F
fengjiayi 已提交
3785 3786 3787
    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.
Y
yuyang18 已提交
3788

J
Jiabin Yang 已提交
3789 3790 3791
        Args:

            throw_on_error (bool): raise Value error when any of required fields is not set.
F
fengjiayi 已提交
3792

J
Jiabin Yang 已提交
3793
            with_details (bool): True if more details about variables and parameters, e.g., :code:`trainable`, :code:`optimize_attr`, need to print.
Y
yuyang18 已提交
3794

H
haowang101779990 已提交
3795
        Returns:
J
Jiabin Yang 已提交
3796
            str: The debug string describe current Program.
Y
yuyang18 已提交
3797 3798

        Raises:
J
Jiabin Yang 已提交
3799
            ValueError: If any of required fields is not set and throw_on_error is True.
F
fengjiayi 已提交
3800

3801 3802 3803 3804 3805 3806 3807
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                prog_string = prog.to_string(throw_on_error=True, with_details=False)
T
tianshuo78520a 已提交
3808
                print("program string without detail: {}".format(prog_string))
J
Jiabin Yang 已提交
3809
                prog_string_with_detail = prog.to_string(throw_on_error=True, with_details=True)
T
tianshuo78520a 已提交
3810
                print("program string with detail: {}".format(prog_string_with_detail))
F
fengjiayi 已提交
3811 3812 3813 3814 3815 3816 3817 3818 3819
        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
            res_str = ""
            for block in self.blocks:
                res_str += block.to_string(throw_on_error, with_details)
        else:
            protostr = self.desc.serialize_to_string()
3820 3821
            proto = framework_pb2.ProgramDesc.FromString(
                six.binary_type(protostr))
F
fengjiayi 已提交
3822 3823
            res_str = _debug_string_(proto, throw_on_error)
        return res_str
3824

W
Wu Yi 已提交
3825
    def _get_desc(self):
Y
yuyang18 已提交
3826 3827 3828 3829 3830 3831 3832
        """
        Get the C++ side of `ProgramDesc` object pointer. The C++ object is
        exposed by :code:`pybind`.

        Notes: This is a very low level API. Users should not use this API
        directly.
        """
3833 3834
        return self.desc

X
version  
Xin Pan 已提交
3835 3836 3837
    def _version(self):
        return self.desc._version()

3838
    @dygraph_not_support
3839
    def clone(self, for_test=False):
Y
yuyang18 已提交
3840
        """
3841
        **Notes**:
J
Jiabin Yang 已提交
3842 3843 3844 3845
            **1.** :code:`Program.clone()` **method DOES NOT clone** :ref:`api_fluid_io_DataLoader` .

            **2. Recommend you to use** :code:`clone` **before using** :code:`Opimizer.minimize`.

3846
            **3. This API has no effect in Dygraph Mode**
Y
yuyang18 已提交
3847

3848 3849
        Create a new Program with forward content of original one when ``for_test=True``.
        Create a new Program as the same as original one when ``for_test=False``
3850

3851

J
Jiabin Yang 已提交
3852
        Some operators, e.g., :ref:`api_fluid_layers_batch_norm` , behave differently between
Y
yuyang18 已提交
3853 3854 3855
        training and testing. They have an attribute, :code:`is_test`, to
        control this behaviour. This method will change the :code:`is_test`
        attribute of them to :code:`True` when :code:`for_test=True`.
3856

Y
yuyang18 已提交
3857
        * Set for_test to False when we want to clone the program for training.
3858
        * Set for_test to True when we want to clone the program for testing.
3859 3860
          We will prune the backward and optimize part of the program when you
          use :code:`clone` after :code:`Opimizer.minimize`, but we still
J
Jiabin Yang 已提交
3861
          recommend you to use :code:`clone` before using :code:`Opimizer.minimize`.
Y
yuyang18 已提交
3862

J
Jiabin Yang 已提交
3863 3864
        For Example:
            .. code-block:: python
L
Luo Tao 已提交
3865

J
Jiabin Yang 已提交
3866 3867 3868 3869
                test_program = fluid.default_main_program().clone(for_test=True)
                # Here we use clone before Momentum
                optimizer = fluid.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
                optimizer.minimize()
3870

J
Jiabin Yang 已提交
3871
        Args:
3872

J
Jiabin Yang 已提交
3873
            for_test (bool): True if change the :code:`is_test` attribute of operators to :code:`True`.
3874

J
Jiabin Yang 已提交
3875 3876
        Returns:
            Program: A new Program with forward content of original one when ``for_test=True``.  A new Program as the same as original one when ``for_test=False``
3877

Y
yuyang18 已提交
3878 3879 3880

        Examples:

J
Jiabin Yang 已提交
3881
        **Notes: The Program's order maybe different after** :code:`clone` **and
3882
        this will not affect your training or testing progress. In the following
J
Jiabin Yang 已提交
3883
        example we give you an simple method** :code:`print_prog(program)` **to
3884
        print Program Descs inorder to make sure you have same print result
J
Jiabin Yang 已提交
3885
        after** :code:`clone`:
3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922
            .. code-block:: python

                import paddle.fluid as fluid
                import six


                def print_prog(prog):
                    for name, value in sorted(six.iteritems(prog.block(0).vars)):
                        print(value)
                    for op in prog.block(0).ops:
                        print("op type is {}".format(op.type))
                        print("op inputs are {}".format(op.input_arg_names))
                        print("op outputs are {}".format(op.output_arg_names))
                        for key, value in sorted(six.iteritems(op.all_attrs())):
                            if key not in ['op_callstack', 'op_role_var']:
                                print(" [ attrs: {}:   {} ]".format(key, value))


        1. To clone a test program, the sample code is:
                .. code-block:: python

                    import paddle.fluid as fluid
                    import six

                    def print_prog(prog):
                        for name, value in sorted(six.iteritems(prog.block(0).vars)):
                            print(value)
                        for op in prog.block(0).ops:
                            print("op type is {}".format(op.type))
                            print("op inputs are {}".format(op.input_arg_names))
                            print("op outputs are {}".format(op.output_arg_names))
                            for key, value in sorted(six.iteritems(op.all_attrs())):
                                if key not in ['op_callstack', 'op_role_var']:
                                    print(" [ attrs: {}:   {} ]".format(key, value))

                    train_program = fluid.Program()
                    startup_program = fluid.Program()
J
Jiabin Yang 已提交
3923 3924 3925

                    # startup_program is used to do some parameter init work,
                    # and main program is used to hold the network
3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936
                    with fluid.program_guard(train_program, startup_program):
                        with fluid.unique_name.guard():
                            img = fluid.layers.data(name='image', shape=[784])
                            hidden = fluid.layers.fc(input=img, size=200, act='relu')
                            hidden = fluid.layers.dropout(hidden, dropout_prob=0.5)
                            loss = fluid.layers.cross_entropy(
                                                      input=fluid.layers.fc(hidden, size=10, act='softmax'),
                                        label=fluid.layers.data(name='label', shape=[1], dtype='int64'))
                            avg_loss = fluid.layers.mean(loss)
                            test_program = train_program.clone(for_test=False)
                    print_prog(test_program)
J
Jiabin Yang 已提交
3937 3938 3939 3940 3941 3942 3943 3944 3945

                    # Due to parameter sharing usage for train and test, so we need to use startup program of train
                    # instead of using test startup program, while nothing is in test's startup program

                    # In Paddle Fluid we will share weights by using the same Variable name. In train and test program
                    # all parameters will have the same name and this can make train and test program sharing parameters,
                    # that's why we need to use startup program of train. And for startup program of test, it has nothing,
                    # since it is a new program.

3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992
                    with fluid.program_guard(train_program, startup_program):
                        with fluid.unique_name.guard():
                            sgd = fluid.optimizer.SGD(learning_rate=1e-3)
                            sgd.minimize(avg_loss)


        2. The clone method can be avoid if you create program for training and program for testing individually.
                .. code-block:: python

                    import paddle.fluid as fluid
                    import six

                    def print_prog(prog):
                        for name, value in sorted(six.iteritems(prog.block(0).vars)):
                            print(value)
                        for op in prog.block(0).ops:
                            print("op type is {}".format(op.type))
                            print("op inputs are {}".format(op.input_arg_names))
                            print("op outputs are {}".format(op.output_arg_names))
                            for key, value in sorted(six.iteritems(op.all_attrs())):
                                if key not in ['op_callstack', 'op_role_var']:
                                    print(" [ attrs: {}:   {} ]".format(key, value))
                    def network(is_test):
                        img = fluid.layers.data(name='image', shape=[784])
                        hidden = fluid.layers.fc(input=img, size=200, act='relu')
                        hidden = fluid.layers.dropout(hidden, dropout_prob=0.5)
                        loss = fluid.layers.cross_entropy(
                            input=fluid.layers.fc(hidden, size=10, act='softmax'),
                            label=fluid.layers.data(name='label', shape=[1], dtype='int64'))
                        avg_loss = fluid.layers.mean(loss)
                        return avg_loss


                    train_program_2 = fluid.Program()
                    startup_program_2 = fluid.Program()
                    test_program_2 = fluid.Program()
                    with fluid.program_guard(train_program_2, startup_program_2):
                        with fluid.unique_name.guard():
                             sgd = fluid.optimizer.SGD(learning_rate=1e-3)
                             sgd.minimize(avg_loss)
                    # the test startup program is not used.
                    with fluid.program_guard(test_program_2, fluid.Program()):
                        with fluid.unique_name.guard():
                            loss = network(is_test=True)
                    print(test_program_2)

        The two code snippets above will generate and print same programs.
3993 3994
        """
        if for_test:
3995
            if self._appending_grad_times > 0:
3996 3997 3998 3999 4000 4001 4002
                forward_prog = Program()
                forward_prog.desc = core.prune_backward(self.desc)
                forward_prog.blocks = [
                    Block(forward_prog, i)
                    for i in six.moves.range(forward_prog.desc.num_blocks())
                ]
                forward_prog._sync_with_cpp()
4003 4004 4005
                p = forward_prog._inference_optimize(prune_read_op=False)
            else:
                p = self._inference_optimize(prune_read_op=False)
4006
        else:
4007
            p = Program()
G
gongweibao 已提交
4008 4009
            p.current_block_idx = self.current_block_idx
            p._seed = self._seed
4010
            p.desc = core.ProgramDesc(self.desc)
M
minqiyang 已提交
4011 4012 4013
            p.blocks = [
                Block(p, i) for i in six.moves.range(self.desc.num_blocks())
            ]
G
gongweibao 已提交
4014 4015

            p._current_role = self._current_role
4016
            p.__op_role_var = self.__op_role_var
4017
            p._appending_grad_times = self._appending_grad_times
G
gongweibao 已提交
4018

W
Wu Yi 已提交
4019
            p._sync_with_cpp()
4020

W
Wu Yi 已提交
4021
        p._copy_param_info_from(self)
W
Wu Yi 已提交
4022
        p._copy_data_info_from(self)
4023
        p._copy_dist_param_info_from(self)
Y
Yu Yang 已提交
4024
        return p
4025

4026
    def _prune(self, targets):
Y
yuyang18 已提交
4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039
        """
        Prune operators and variables which are not needed to generate
        :code:`targets`.

        Notes: This is a very low level API. Users should not use this API
        directly. This API is in flux and not stable.

        Args:
            targets(list|Variable|Operator): A list of variables or operators
                need to be pruned

        Returns:
            Program:  A new, pruned program.
4040 4041 4042 4043
        """

        if not isinstance(targets, list):
            targets = [targets]
Y
yuyang18 已提交
4044

4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078
        targets_idx = []
        for t in targets:
            if not isinstance(t, Operator):
                if isinstance(t, Variable):
                    # After transpiler processing, the op that output this
                    # variable maybe has been changed, so t.op is not reliable
                    # and we need to find the current op that generate this
                    # variable here.
                    t.op = None
                    global_block = self.global_block()
                    for idx, op in enumerate(global_block.ops):
                        if t.name in op.output_arg_names:
                            t.op = op
                            break

                    t = t.op
                    if t is None:
                        raise ValueError(
                            "The target variable must have an "
                            "associated operator that generates it.")
                else:
                    raise ValueError("All targets of prune() can only be "
                                     "Variable or Operator.")

            targets_idx.append([t.block.idx, t.idx])
        res = Program()
        res.desc = core.prune(self.desc, set(), targets_idx)
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
        res._sync_with_cpp()
        return res

    def _prune_with_input(self, feeded_var_names, targets):
Y
yuyang18 已提交
4079
        """
4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096
        Prune operators and variables which are not needed to generate
        :code:`targets`. Prune operators and variables which are needed 
        to generate feeded_var 

        Notes: This is a very low level API. Users should not use this API
        directly. This API is in flux and not stable.

        Args:
            feeded_var_names(list|str): A list of variable names from where
                pruning start. If it is set as [], this API works just like _prune()
            targets(list|Variable|Operator): A list of variables or operators
                need to be pruned

        Returns:
            Program:  A new, pruned program.
        """

4097 4098
        if not isinstance(feeded_var_names, list):
            feeded_var_names = [feeded_var_names]
4099 4100
        if not isinstance(targets, list):
            targets = [targets]
4101 4102 4103 4104 4105 4106

        for var in feeded_var_names:
            if not isinstance(var, six.string_types):
                raise ValueError("All feeded_var_names of prune() can only be "
                                 "str.")

4107 4108 4109 4110
        targets_idx = []
        for t in targets:
            if not isinstance(t, Operator):
                if isinstance(t, Variable):
4111 4112
                    # After transpiler processing, the op that output this
                    # variable maybe has been changed, so t.op is not reliable
4113
                    # and we need to find the current op that generate this
4114 4115 4116 4117 4118 4119 4120 4121
                    # variable here.
                    t.op = None
                    global_block = self.global_block()
                    for idx, op in enumerate(global_block.ops):
                        if t.name in op.output_arg_names:
                            t.op = op
                            break

4122
                    t = t.op
4123 4124 4125 4126
                    if t is None:
                        raise ValueError(
                            "The target variable must have an "
                            "associated operator that generates it.")
4127
                else:
4128 4129
                    raise ValueError("All targets of prune() can only be "
                                     "Variable or Operator.")
4130 4131 4132

            targets_idx.append([t.block.idx, t.idx])
        res = Program()
4133
        res.desc = core.prune(self.desc, set(feeded_var_names), targets_idx)
M
minqiyang 已提交
4134 4135 4136
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
W
Wu Yi 已提交
4137
        res._sync_with_cpp()
4138 4139
        return res

X
Xin Pan 已提交
4140
    def _inference_optimize(self, prune_read_op=True):
Y
yuyang18 已提交
4141
        """
F
fengjiayi 已提交
4142 4143 4144 4145 4146
        This method will create a new program and do following adjustments on it:
        1. Remove all reader variables and their creator ops if exist.

        2. Remove the :code:`read_op` if exists.

4147
        3. change the :code:`is_test`
Y
yuyang18 已提交
4148 4149 4150
        attribute of operators to :code:`True`. All the :code:`Parameter`
        information will be lost.

4151
        Args:
X
Xin Pan 已提交
4152 4153
            prune_read_op(bool): remove the read ops that are added by py_reader
                                 for cpp inference library
4154

Y
yuyang18 已提交
4155 4156 4157 4158 4159 4160
        Notes: This API is a very low level API. Use
        :code:`Program.clone(for_test=True)` instead.

        Returns:
            Program: The new program.
        """
4161
        res = Program()
4162
        res.desc = core.ProgramDesc(self.desc)
F
fengjiayi 已提交
4163 4164 4165 4166

        # remove all readers and the read_op if exist
        read_op_idx = 0
        root_block = res.desc.block(0)
X
Xin Pan 已提交
4167
        if prune_read_op:
4168 4169 4170 4171 4172 4173 4174 4175 4176
            while True:
                if read_op_idx >= root_block.op_size() or root_block.op(
                        read_op_idx).type() == 'read':
                    break
                read_op_idx += 1
            if read_op_idx < root_block.op_size():
                root_block._remove_op(0, read_op_idx + 1)
            for var in root_block.all_vars():
                if var.type() == core.VarDesc.VarType.READER:
M
minqiyang 已提交
4177
                    root_block._remove_var(cpt.to_bytes(var.name()))
F
fengjiayi 已提交
4178 4179

        # change all `is_test` attributes to True
M
minqiyang 已提交
4180
        for i in six.moves.range(res.desc.num_blocks()):
4181
            block = res.desc.block(i)
M
minqiyang 已提交
4182
            for j in six.moves.range(block.op_size()):
4183 4184
                op = block.op(j)
                if op.has_attr('is_test'):
W
Wu Yi 已提交
4185
                    op._set_attr('is_test', True)
M
minqiyang 已提交
4186 4187 4188
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
W
Wu Yi 已提交
4189
        res._sync_with_cpp()
4190 4191
        return res

4192 4193
    @staticmethod
    def parse_from_string(binary_str):
Y
yuyang18 已提交
4194
        """
J
Jiabin Yang 已提交
4195 4196 4197 4198
        **Notes**:
            **1. All information about parameters will be lost after serialization**

            **2. This API has no effect in Dygraph mode**
Y
yuyang18 已提交
4199

4200 4201
        Deserialize a Program from  `protobuf <https://en.wikipedia.org/wiki/Protocol_Buffers>`_  binary string.
        This method always use to save and load model
Y
yuyang18 已提交
4202

J
Jiabin Yang 已提交
4203
        Args:
Y
yuyang18 已提交
4204

J
Jiabin Yang 已提交
4205
            binary_str_type (str): the binary prootbuf string.
4206

J
Jiabin Yang 已提交
4207 4208
        Returns:
            Program: A deserialized Program.
4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                startup_prog = fluid.Program()
                main_prog = fluid.Program()
                with fluid.program_guard(startup_prog, main_prog):
                    x = fluid.layers.data(
                        name='X', shape=[1000, 784], dtype='float32', append_batch_size=False)

                    y = fluid.layers.data(
                        name='Y', shape=[784, 100], dtype='float32', append_batch_size=False)

                    z = fluid.layers.mul(x=x, y=y)

                    binary_str = fluid.default_main_program().desc.serialize_to_string()
                    prog_restored = fluid.default_main_program().parse_from_string(binary_str)

                    print(fluid.default_main_program())
                    print(prog_restored)
Y
yuyang18 已提交
4231
        """
4232 4233
        p = Program()
        p.desc = core.ProgramDesc(binary_str)
M
minqiyang 已提交
4234
        p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())]
W
Wu Yi 已提交
4235
        p._sync_with_cpp()
4236
        return p
Y
Yu Yang 已提交
4237

4238
    @staticmethod
4239
    def _construct_from_desc(desc):
4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254
        """
        Construct a program from program desc.

        Args:
            desc(core.ProgramDesc): The program desc for constructing.

        Returns:
            Program: A program.
        """
        p = Program()
        p.desc = desc
        p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())]
        p._sync_with_cpp()
        return p

D
dzhwinter 已提交
4255 4256
    @property
    def random_seed(self):
Y
yuyang18 已提交
4257
        """
J
Jiabin Yang 已提交
4258
        The default random seed for random operators in Program. ``0`` means get
Y
yuyang18 已提交
4259 4260
        the random seed from random device.

J
Jiabin Yang 已提交
4261 4262 4263 4264
        **Notes: It must be set before the operators have been added.**

        Returns:
            int64: Random seed in current Program
4265

4266 4267 4268 4269 4270 4271 4272 4273

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                random_seed = prog.random_seed
4274 4275 4276
                x_var = fluid.layers.data(name="X", shape=[3,3], dtype="float32", append_batch_size=False)

                # Here we need to set random seed before we use fluid.layers.dropout
4277 4278
                print(random_seed)
                prog.random_seed = 1
4279 4280
                z_var = fluid.layers.dropout(x_var, 0.7)

4281
                print(prog.random_seed)
Y
yuyang18 已提交
4282
        """
D
dzhwinter 已提交
4283 4284
        return self._seed

Q
qiaolongfei 已提交
4285 4286
    @property
    def num_blocks(self):
Y
yuyang18 已提交
4287
        """
4288 4289
        The number of :ref:`api_guide_Block_en`  in this Program.

J
Jiabin Yang 已提交
4290 4291 4292 4293
        **Notes: This API has no effect in Dygraph mode**

        Returns:
            int(Platform-dependent size): num of :ref:`api_guide_Block_en`  in current Program
4294

4295 4296 4297 4298 4299 4300 4301 4302 4303

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                num_blocks = prog.num_blocks
                print(num_blocks)
4304 4305


Y
yuyang18 已提交
4306
        """
Q
qiaolongfei 已提交
4307 4308
        return self.desc.num_blocks()

D
dzhwinter 已提交
4309 4310 4311 4312 4313 4314
    @random_seed.setter
    def random_seed(self, seed):
        if not isinstance(seed, int):
            raise ValueError("Seed must be a integer.")
        self._seed = seed

Y
Yu Yang 已提交
4315
    def __repr__(self):
4316
        return self.__str__()
4317

Y
Yu Yang 已提交
4318
    def global_block(self):
Y
yuyang18 已提交
4319
        """
J
Jiabin Yang 已提交
4320 4321
        **Notes**:
            **This API has no effect in Dygraph mode**
4322 4323 4324

        Get the first :ref:`api_guide_Block_en` of this Program.

J
Jiabin Yang 已提交
4325 4326
        Returns:
            :ref:`api_guide_Block_en`: The first :ref:`api_guide_Block_en`  of this Program.
4327

4328 4329 4330 4331 4332 4333 4334 4335 4336

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                gb_block = prog.global_block()
                print(gb_block)
4337

Y
yuyang18 已提交
4338
        """
Y
Yu Yang 已提交
4339 4340
        return self.blocks[0]

Q
Qiao Longfei 已提交
4341
    def block(self, index):
Y
yuyang18 已提交
4342
        """
J
Jiabin Yang 已提交
4343 4344
        **Notes**:
            **This API has no effect in Dygraph mode**
Y
yuyang18 已提交
4345

4346 4347
        Get the :code:`index`  :ref:`api_guide_Block_en`  of this Program

J
Jiabin Yang 已提交
4348 4349
        Args:
            index (int) - The index of  :ref:`api_guide_Block_en`  to get
4350

J
Jiabin Yang 已提交
4351 4352
        Returns:
            :ref:`api_guide_Block_en`: The :code:`index` block
4353 4354 4355 4356 4357 4358 4359 4360 4361

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                block_0 = prog.block(0)
                print(block_0)
Y
yuyang18 已提交
4362
        """
Q
Qiao Longfei 已提交
4363 4364
        return self.blocks[index]

Y
Yu Yang 已提交
4365
    def current_block(self):
Y
yuyang18 已提交
4366
        """
J
Jiabin Yang 已提交
4367 4368
        **Notes**:
            **This API has no effect in Dygraph mode**
4369

J
Jiabin Yang 已提交
4370 4371
        Get the current  :ref:`api_guide_Block_en` . The :code:`current`  :ref:`api_guide_Block_en`
        is the  :ref:`api_guide_Block_en`  to append operators.
4372

J
Jiabin Yang 已提交
4373 4374
        Returns:
             :ref:`api_guide_Block_en`: The :code:`index`  :ref:`api_guide_Block_en`
4375

4376 4377 4378 4379 4380 4381 4382 4383
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                current_blk = prog.current_block()
                print(current_blk)
Y
yuyang18 已提交
4384
        """
Y
Yu Yang 已提交
4385 4386
        return self.blocks[self.current_block_idx]

W
Wu Yi 已提交
4387
    def _create_block(self, parent_idx=None):
Y
yuyang18 已提交
4388 4389 4390 4391 4392
        """
        Create a new block with the :code:`parent_idx` and change the current block
        to new block.

        Args:
J
Jiabin Yang 已提交
4393

Y
yuyang18 已提交
4394 4395 4396 4397 4398
            parent_idx(int): The parent block index.

        Returns:
            Block: The new block.
        """
Y
Yu Yang 已提交
4399
        new_block_idx = len(self.blocks)
F
update  
fengjiayi 已提交
4400 4401 4402
        parent = self.current_block() if parent_idx is None else self.block(
            parent_idx)
        self.desc.append_block(parent.desc)
Y
Yu Yang 已提交
4403 4404 4405 4406
        self.current_block_idx = new_block_idx
        self.blocks.append(Block(self, self.current_block_idx))
        return self.current_block()

W
Wu Yi 已提交
4407
    def _rollback(self):
Y
yuyang18 已提交
4408 4409 4410 4411 4412
        """
        Exit a code block, i.e., roll back to the parent block.
        Returns:
            None
        """
Y
Yu Yang 已提交
4413 4414
        self.current_block_idx = self.current_block().parent_idx

W
Wu Yi 已提交
4415
    def _sync_with_cpp(self):
Y
yuyang18 已提交
4416 4417 4418 4419 4420 4421 4422 4423 4424 4425
        """
        Synchronize Python instance to its binding C++ object instance.
        If the program is modified in C++ space, this method should be invoked.

        Notes: This is a very low level API. Users should not invoke it
        directly.

        Returns:
            None
        """
Q
Qiao Longfei 已提交
4426 4427 4428
        for block_idx in range(len(self.blocks), self.desc.num_blocks()):
            self.blocks.append(Block(self, block_idx))
        for block in self.blocks:
W
Wu Yi 已提交
4429
            block._sync_with_cpp()
Q
Qiao Longfei 已提交
4430

W
Wu Yi 已提交
4431
    def _copy_param_info_from(self, other):
4432
        """
4433
        Copy the information of parameters from other program.
D
dzhwinter 已提交
4434

Y
yuyang18 已提交
4435 4436 4437
        Notes: This is a very low level API. Users should not invoke it
        directly.

4438 4439 4440 4441 4442 4443 4444
        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
W
Wu Yi 已提交
4445
            raise TypeError("_copy_param_info_from should be invoked with "
4446 4447 4448
                            "Program")

        if len(self.blocks) != len(other.blocks):
W
Wu Yi 已提交
4449
            raise ValueError("_copy_param_info_from should be invoked with two "
4450
                             "program, with represent the same topology")
W
Wu Yi 已提交
4451
        self.global_block()._copy_param_info_from(other.global_block())
4452

4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467
    def _copy_dist_param_info_from(self, other):
        """
        Copy the information of distributed information from other program.

        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
            raise TypeError("_copy_dist_param_info_from should be invoked with "
                            "Program")
        self._is_distributed = other._is_distributed
        self._is_chief = other._is_chief
4468
        self._parameters_on_pservers = other._parameters_on_pservers
4469
        self._endpoints = other._endpoints
4470
        self._ps_endpoint = other._ps_endpoint
4471 4472
        self._distributed_lookup_table = other._distributed_lookup_table

W
Wu Yi 已提交
4473
    def _copy_data_info_from(self, other):
F
fengjiayi 已提交
4474 4475
        """
        Copy the information of data variables from other program.
D
dzhwinter 已提交
4476

Y
yuyang18 已提交
4477 4478 4479
        Notes: This is a very low level API. Users should not invoke it
        directly.

F
fengjiayi 已提交
4480 4481 4482 4483 4484 4485 4486
        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
4487
            raise TypeError("_copy_data_info_from should be invoked with "
F
fengjiayi 已提交
4488 4489 4490
                            "Program")

        if len(self.blocks) != len(other.blocks):
4491
            raise ValueError("_copy_data_info_from should be invoked with two "
F
fengjiayi 已提交
4492
                             "program, with represent the same topology")
4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505

        # NOTE(zhiqiu): All vars in cloned program exist in original program.
        # The reverse is not true, due to backward pruning.
        for i, block in enumerate(other.blocks):
            for var in list(block.vars.values()):
                if not self.blocks[i].has_var(var.name):
                    continue
                if var.is_data:
                    self.blocks[i].var(var.name).is_data = True
                if var.desc.need_check_feed():
                    self.blocks[i].var(var.name).desc.set_need_check_feed(True)
                if var.stop_gradient:
                    self.blocks[i].var(var.name).stop_gradient = True
F
fengjiayi 已提交
4506

4507
    @dygraph_not_support
4508
    def list_vars(self):
Y
yuyang18 已提交
4509
        """
J
Jiabin Yang 已提交
4510
        Get all :ref:`api_guide_Variable_en` from this Program. A iterable object is returned.
Y
yuyang18 已提交
4511

J
Jiabin Yang 已提交
4512 4513
        Returns:
            iterable :ref:`api_guide_Variable_en`: The Generator will yield every variable in this program.
4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                img = fluid.layers.data(name='img', shape=[1,28,28], dtype='float32')
                label = fluid.layers.data(name='label', shape=[128,1], dtype='int64')
                for var in prog.list_vars():
                    print(var)
Y
yuyang18 已提交
4525
        """
4526
        for each_block in self.blocks:
4527
            for each_var in list(each_block.vars.values()):
4528 4529
                yield each_var

4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588
    @dygraph_not_support
    def all_parameters(self):
        """
        Get all :ref:`api_guide_parameter_en` from this Program. A list object is returned.

        Returns:
            list[ :ref:`api_guide_parameter_en` ]: The list contians all parameters in this program.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                program = fluid.default_main_program()
                data = fluid.data(name='x', shape=[None, 13], dtype='float32')
                hidden = fluid.layers.fc(input=data, size=10)
                loss = fluid.layers.mean(hidden)
                fluid.optimizer.SGD(learning_rate=0.01).minimize(loss)

                for param in program.all_parameters():
                    print(param)

                # Here will print all parameters in current program, in this example,
                # the result is like:
                #
                # name: "fc_0.w_0"
                # type {
                #   type: LOD_TENSOR
                #   lod_tensor {
                #     tensor {
                #       data_type: FP32
                #       dims: 13
                #       dims: 10
                #     }
                #   }
                # }
                # persistable: true
                #
                # name: "fc_0.b_0"
                # type {
                # type: LOD_TENSOR
                # lod_tensor {
                #     tensor {
                #       data_type: FP32
                #       dims: 10
                #     }
                #   }
                # }
                # persistable: true
                #
                # Here print(param) will print out all the properties of a parameter,
                # including name, type and persistable, you can access to specific
                # property of a parameter, such as param.name, param.type
        """
        parameters = []
        for each_block in self.blocks:
            parameters.extend(each_block.all_parameters())
        return parameters

Y
Yu Yang 已提交
4589

4590
@six.add_metaclass(ParameterMetaClass)
Y
Yu Yang 已提交
4591
class Parameter(Variable):
4592
    """
4593
    Parameter is derived from Variable. A parameter is a persistable
4594
    Variable, and will be updated by optimizers after each iteration.
4595
    The training of a neural network is essentially the updating of
4596 4597
    its parameters.

4598
    Relative to a general Variable, a Parameter has several its own
4599 4600
    member variables:

4601 4602 4603 4604 4605 4606 4607 4608
    Args:
        trainable(bool): True if the parameter need to be updated after
            iterations.
        optimize_attr(map): Parameter attributes related with optimizing.
            Currently, it only contains 'learning_rate'.
            Default: {'learning_rate': 1.0}
        regularizer(WeightDecayRegularizer): The Regularizer which will
            be applied on the parameter. Default: None
T
tianshuo78520a 已提交
4609
        gradient_clip_attr(BaseGradientClipAttr): The gradient clip strategy
4610 4611 4612
            which will be applied on the parameter. Default: None
        do_model_average(bool): True if the model average strategy will
            be applied on this parameter.
4613 4614
    """

4615 4616 4617 4618 4619 4620
    def __init__(self,
                 block,
                 shape,
                 dtype,
                 type=core.VarDesc.VarType.LOD_TENSOR,
                 **kwargs):
4621 4622 4623 4624 4625
        if shape is None:
            raise ValueError("The shape of Parameter should not be None")
        if dtype is None:
            raise ValueError("The dtype of Parameter should not be None")

Y
Yu Yang 已提交
4626
        if len(shape) == 0:
4627 4628
            raise ValueError(
                "The dimensions of shape for Parameter must be greater than 0")
Y
Yu Yang 已提交
4629 4630 4631

        for each in shape:
            if each < 0:
4632 4633 4634
                raise ValueError(
                    "Each dimension of shape for Parameter must be greater than 0, but received %s"
                    % list(shape))
4635 4636

        Variable.__init__(
4637 4638 4639 4640 4641 4642 4643
            self,
            block,
            persistable=True,
            shape=shape,
            dtype=dtype,
            type=type,
            **kwargs)
Y
Yu Yang 已提交
4644 4645 4646 4647
        self.trainable = kwargs.get('trainable', True)

        self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})

4648 4649
        self.regularizer = kwargs.get('regularizer', None)

F
fengjiayi 已提交
4650
        self.gradient_clip_attr = kwargs.get('gradient_clip_attr', None)
Y
Yu Yang 已提交
4651

W
wanghaoshuang 已提交
4652
        self.do_model_average = kwargs.get('do_model_average', None)
W
wanghaoshuang 已提交
4653

4654 4655
        self.is_distributed = False

F
fengjiayi 已提交
4656 4657 4658
    def __str__(self):
        return self.to_string(True)

F
update  
fengjiayi 已提交
4659 4660 4661
    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.
D
dzhwinter 已提交
4662

F
update  
fengjiayi 已提交
4663 4664 4665 4666 4667 4668 4669 4670
        Args:
            throw_on_error(bool): raise exception when self is not initialized
                when throw_on_error is True
            with_details(bool): more details about variables and parameters
                (e.g. trainable, optimize_attr, ...) will be printed when with_details is True

        Returns(str): The debug string.

4671 4672 4673 4674 4675 4676 4677 4678 4679
        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                rlt = fluid.layers.data("fake_data", shape=[1,1], dtype='float32')
                debug_str = prog.to_string(throw_on_error=True, with_details=False)
                print(debug_str)
F
update  
fengjiayi 已提交
4680 4681 4682 4683 4684 4685
        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
            res_str = Variable.to_string(self, throw_on_error, True)
            additional_attr = ("trainable", "optimize_attr", "regularizer",
W
wanghaoshuang 已提交
4686
                               "gradient_clip_attr", "do_model_average")
F
update  
fengjiayi 已提交
4687
            for attr_name in additional_attr:
4688 4689
                res_str += "%s: %s\n" % (attr_name,
                                         cpt.to_text(getattr(self, attr_name)))
F
update  
fengjiayi 已提交
4690 4691
        else:
            res_str = Variable.to_string(self, throw_on_error, False)
F
fengjiayi 已提交
4692 4693 4694 4695
        return res_str

    __repr__ = __str__

Y
Yu Yang 已提交
4696

4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714
class ParamBase(core.VarBase):
    """
    ParamBase is derived from VarBase( Which is the Variable in Dygraph Mode ). A ParamBase is a persistable
    VarBase, and will be updated by optimizers after each iteration.
    The training of a neural network is essentially the updating of
    its ParamBase.

    Relative to a general Variable, a ParamBase has several its own
    member variables:

    Args:
        trainable(bool): True if the ParamBase need to be updated after
            iterations.
        optimize_attr(map): ParamBase attributes related with optimizing.
            Currently, it only contains 'learning_rate'.
            Default: {'learning_rate': 1.0}
        regularizer(WeightDecayRegularizer): The Regularizer which will
            be applied on the ParamBase. Default: None
T
tianshuo78520a 已提交
4715
        gradient_clip_attr(BaseGradientClipAttr): The gradient clip strategy
4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760
            which will be applied on the ParamBase. Default: None
        do_model_average(bool): True if the model average strategy will
            be applied on this ParamBase.
    """

    @dygraph_only
    def __init__(self, shape, dtype, **kwargs):
        if shape is None:
            raise ValueError("The shape of Parameter should not be None")
        if dtype is None:
            raise ValueError("The dtype of Parameter should not be None")

        if len(shape) == 0:
            raise ValueError(
                "The dimensions of shape for Parameter must be greater than 0")

        for each in shape:
            if each < 0:
                raise ValueError(
                    "Each dimension of shape for Parameter must be greater than 0, but received %s"
                    % list(shape))

        if dtype is not None:
            if not isinstance(dtype, core.VarDesc.VarType):
                dtype = convert_np_dtype_to_dtype_(dtype)

        name = kwargs.get('name', unique_name.generate('_param_base'))

        super(ParamBase, self).__init__(dtype
                                        if dtype else core.VarDesc.VarType.FP32,
                                        list(shape) if shape else [], name,
                                        core.VarDesc.VarType.LOD_TENSOR, True)

        self.trainable = kwargs.get('trainable', True)

        self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})

        self.regularizer = kwargs.get('regularizer', None)

        self.gradient_clip_attr = kwargs.get('gradient_clip_attr', None)

        self.do_model_average = kwargs.get('do_model_average', None)

        self.is_distributed = False

4761
        # self.block = default_main_program().global_block()
4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799

    def __str__(self):
        return self.to_string(True)

    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.

        Args:
            throw_on_error(bool): raise exception when self is not initialized
                when throw_on_error is True
            with_details(bool): more details about variables and parameters
                (e.g. trainable, optimize_attr, ...) will be printed when with_details is True

        Returns(str): The debug string.

        Examples:
            .. code-block:: python

                import paddle.fluid as fluid

                prog = fluid.default_main_program()
                rlt = fluid.layers.data("fake_data", shape=[1,1], dtype='float32')
                debug_str = prog.to_string(throw_on_error=True, with_details=False)
                print(debug_str)
        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        tensor = self.value().get_tensor()
        if tensor._is_initialized():
            return 'name %s, dtype: %s shape: %s %s' % (self.name, self.dtype,
                                                        self.shape, str(tensor))
        else:
            return 'name %s, shape: %s, not inited' % (self.name, self.shape)

    __repr__ = __str__


Y
Yu Yang 已提交
4800
# program is a global instance.
Y
Yu Yang 已提交
4801 4802
_main_program_ = Program()
_startup_program_ = Program()
4803

4804

4805
def default_startup_program():
Y
Yu Yang 已提交
4806
    """
Y
yuyang18 已提交
4807 4808
    Get default/global startup program.

J
Jiabin Yang 已提交
4809 4810 4811
    The layer function in :ref:`api_fluid_layers` will create parameters, :ref:`api_paddle_data_reader_reader` ,
    `NCCL <https://developer.nvidia.com/nccl>`_ handles as global variables. The :code:`startup_program` will
    initialize them by the OPs in startup  :ref:`api_fluid_Program` . The  :ref:`api_fluid_layers`  function will
Y
yuyang18 已提交
4812 4813 4814
    append these initialization operators into startup program.

    This method will return the :code:`default` or the :code:`current` startup
J
Jiabin Yang 已提交
4815
    program. Users can use  :ref:`api_fluid_program_guard`  to switch :ref:`api_fluid_Program` .
4816

J
Jiabin Yang 已提交
4817
    Returns: current default startup :ref:`api_fluid_Program`
4818

J
Jiabin Yang 已提交
4819
    Returns type: :ref:`api_fluid_Program`
4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid

            main_program = fluid.Program()
            startup_program = fluid.Program()
            with fluid.program_guard(main_program=main_program, startup_program=startup_program):
                x = fluid.layers.data(name="x", shape=[-1, 784], dtype='float32')
                y = fluid.layers.data(name="y", shape=[-1, 1], dtype='int32')
                z = fluid.layers.fc(name="fc", input=x, size=10, act="relu")

                print("main program is: {}".format(fluid.default_main_program()))
                print("start up program is: {}".format(fluid.default_startup_program()))
Y
Yu Yang 已提交
4835
    """
Y
Yu Yang 已提交
4836
    return _startup_program_
4837

4838

4839
def default_main_program():
Y
Yu Yang 已提交
4840
    """
4841 4842 4843 4844 4845
    This API can be used to get ``default main program`` which store the 
    descriptions of ``op`` and ``variable``.
    
    For example ``z = fluid.layers.elementwise_add(x, y)`` will create a new ``elementwise_add`` 
    ``op`` and a new ``z`` ``variable``, and they will be recorded in ``default main program`` 
Y
yuyang18 已提交
4846

4847 4848
    The ``default_main_program`` is the default value for ``Program`` parameter in 
    a lot of ``fluid`` APIs. For example, the :code:`Executor.run()` will execute the
Y
yuyang18 已提交
4849
    :code:`default_main_program` when the program is not specified.
4850

4851 4852
    If you want to replace the ``default main program``, you can use :ref:`api_fluid_program_guard`
    
Y
Yu Yang 已提交
4853
    Returns:
4854
        :ref:`api_fluid_Program`: a ``Program`` which holding the descriptions of ops and variables in the network.
4855 4856 4857 4858 4859

    Examples:
        ..  code-block:: python

            import paddle.fluid as fluid
4860

4861
            # Sample Network:
4862 4863
            data = fluid.data(name='image', shape=[None, 3, 224, 224], dtype='float32')
            label = fluid.data(name='label', shape=[None, 1], dtype='int64')
4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882
            
            conv1 = fluid.layers.conv2d(data, 4, 5, 1, act=None)
            bn1 = fluid.layers.batch_norm(conv1, act='relu')
            pool1 = fluid.layers.pool2d(bn1, 2, 'max', 2)
            conv2 = fluid.layers.conv2d(pool1, 16, 5, 1, act=None)
            bn2 = fluid.layers.batch_norm(conv2, act='relu')
            pool2 = fluid.layers.pool2d(bn2, 2, 'max', 2)
            
            fc1 = fluid.layers.fc(pool2, size=50, act='relu')
            fc2 = fluid.layers.fc(fc1, size=102, act='softmax')
            
            loss = fluid.layers.cross_entropy(input=fc2, label=label)
            loss = fluid.layers.mean(loss)
            opt = fluid.optimizer.Momentum(
                learning_rate=0.1,
                momentum=0.9,
                regularization=fluid.regularizer.L2Decay(1e-4))
            opt.minimize(loss)
            
4883
            #print the number of blocks in the program, 1 in this case
4884
            print(fluid.default_main_program().num_blocks)
4885 4886

            #print the description of variable 'image'
4887
            print(fluid.default_main_program().blocks[0].var('image'))
4888

Y
Yu Yang 已提交
4889
    """
Y
Yu Yang 已提交
4890
    return _main_program_
Y
Yu Yang 已提交
4891 4892 4893 4894 4895


def switch_main_program(program):
    """
    Switch the main program to a new program.
4896

Y
Yu Yang 已提交
4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910
    Args:
        program(Program): The new main program

    Returns:
        Program: The previous main program
    """
    global _main_program_
    prev_program = _main_program_
    _main_program_ = program
    return prev_program


def switch_startup_program(program):
    """
4911
    Switch the startup program to a new program
Y
Yu Yang 已提交
4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923
    Args:
        program(Program): The new startup program

    Returns:
        Program: The previous startup program
    """
    global _startup_program_
    prev_program = _startup_program_
    _startup_program_ = program
    return prev_program


S
rename  
sneaxiy 已提交
4924
@signature_safe_contextmanager
Y
Yu Yang 已提交
4925 4926
def program_guard(main_program, startup_program=None):
    """
4927 4928
    Change the global main program and startup program with `"with"` statement.
    Layer functions in the Python `"with"` block will append operators and
Y
yuyang18 已提交
4929
    variables to the new main programs.
4930

G
guofei 已提交
4931 4932 4933 4934 4935 4936 4937
    Args:
        main_program(Program): New main program inside `"with"` statement.
        startup_program(Program, optional): New startup program inside `"with"` 
            statement. :code:`None` means not changing startup program, 
            default_startup_program is still used.
            Default: None.

Y
Yu Yang 已提交
4938
    Examples:
4939 4940 4941
       .. code-block:: python
       
         import paddle.fluid as fluid
Y
yuyang18 已提交
4942

4943 4944 4945
         main_program = fluid.Program()
         startup_program = fluid.Program()
         with fluid.program_guard(main_program, startup_program):
G
guofei 已提交
4946
             data = fluid.data(name='image', shape=[None, 784, 784], dtype='float32')
4947
             hidden = fluid.layers.fc(input=data, size=10, act='relu')
Y
yuyang18 已提交
4948 4949 4950

    Notes: The temporary :code:`Program` can be used if the user does not need
    to construct either of startup program or main program.
4951

Y
Yu Yang 已提交
4952
    Examples:
4953
       .. code-block:: python
Y
yuyang18 已提交
4954

4955 4956 4957 4958 4959
         import paddle.fluid as fluid

         main_program = fluid.Program()
         # does not care about startup program. Just pass a temporary value.
         with fluid.program_guard(main_program, fluid.Program()):
G
guofei 已提交
4960 4961
             data = fluid.data(name='image', shape=[None, 784, 784], dtype='float32')
    
Y
Yu Yang 已提交
4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973
    """
    if not isinstance(main_program, Program):
        raise TypeError("main_program should be Program")
    main_program = switch_main_program(main_program)
    if startup_program is not None:
        if not isinstance(startup_program, Program):
            raise TypeError("startup_program should be Program")
        startup_program = switch_startup_program(startup_program)
    yield
    switch_main_program(main_program)
    if startup_program is not None:
        switch_startup_program(startup_program)
X
xuwei06 已提交
4974 4975


W
Wu Yi 已提交
4976
def _get_var(name, program=None):
X
xuwei06 已提交
4977
    """
Y
yuyang18 已提交
4978
    Get a variable by name from the global block of a program.
F
fengjiayi 已提交
4979

X
xuwei06 已提交
4980 4981 4982
    Args:
        name(str): name of the variable
        program(Program|None): program object.
T
tangwei12 已提交
4983
        If None, default_global_program() will be used.
X
xuwei06 已提交
4984 4985 4986 4987 4988 4989 4990

    Returns:
        Variable
    """
    if program is None:
        program = default_main_program()
    assert isinstance(name, str)
4991
    assert isinstance(program, Program)
X
xuwei06 已提交
4992 4993

    return program.global_block().var(name)
4994 4995


S
rename  
sneaxiy 已提交
4996
@signature_safe_contextmanager
L
lujun 已提交
4997 4998 4999 5000
def _dygraph_guard(tracer):
    global _dygraph_tracer_
    tmp_trace = _dygraph_tracer_
    _dygraph_tracer_ = tracer
5001
    core._switch_tracer(tracer)
M
minqiyang 已提交
5002

5003
    yield
P
Paddle CI 已提交
5004

5005
    core._switch_tracer(tmp_trace)
L
lujun 已提交
5006
    _dygraph_tracer_ = tmp_trace
P
Paddle CI 已提交
5007 5008


S
rename  
sneaxiy 已提交
5009
@signature_safe_contextmanager
L
lujun 已提交
5010 5011 5012 5013
def _dygraph_place_guard(place):
    global _dygraph_current_expected_place_
    tmp_place = _dygraph_current_expected_place_
    _dygraph_current_expected_place_ = place
M
minqiyang 已提交
5014

5015
    yield
M
minqiyang 已提交
5016

L
lujun 已提交
5017
    _dygraph_current_expected_place_ = tmp_place
5018 5019 5020 5021 5022 5023 5024


def load_op_library(lib_filename):
    """
    Load a dynamic library, including custom operators and kernels.
    When library is loaded, ops and kernels registered in the library
    will be available in PaddlePaddle main process.
T
tianshuo78520a 已提交
5025
    Please note, the type of custom operators can't have the same type
5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039
    with the existing operators in the framework.

    Args:
        lib_filename (str): name of dynamic library.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            #fluid.load_op_library('custom_op.so')

    """
    core.load_op_library(lib_filename)
    OpProtoHolder.instance().update_op_proto()