framework.py 76.2 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

Y
Yu Yang 已提交
17
import collections
X
Xin Pan 已提交
18
from collections import defaultdict
Q
qiaolongfei 已提交
19
import contextlib
P
peizhilin 已提交
20
import os
F
fengjiayi 已提交
21
import re
22
import traceback
23
import six
24

Y
Yu Yang 已提交
25
import numpy as np
Q
qiaolongfei 已提交
26

M
minqiyang 已提交
27
from .. import compat as cpt
28
from .proto import framework_pb2
29
try:
P
peizhilin 已提交
30
    if os.name == 'nt':
P
peizhilin 已提交
31
        import sys
P
peizhilin 已提交
32 33 34 35 36
        third_lib_path = os.path.abspath(os.path.dirname(
            __file__)) + os.sep + '..' + os.sep + 'libs'
        os.environ['path'] += ';' + third_lib_path
        sys.path.append(third_lib_path)

37
    from . import core
38
except ImportError as e:
P
peizhilin 已提交
39 40 41 42 43 44 45 46 47 48 49 50
    if os.name == 'nt':
        raise ImportError(
            """NOTE: You may need to run \"set PATH=c:\python27\lib:%PATH%\"
        if you encounters \"mkldnn.dll not found\" errors. If you have python
        installed in other directory, replace \"c:\python27\lib" with your own
        directory. The original error is: \n""" + cpt.get_exception_message(e))
    else:
        raise ImportError(
            """NOTE: You may need to run \"export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH\"
        if you encounters \"libmkldnn.so not found\" errors. If you have python
        installed in other directory, replace \"/usr/local/lib\" with your own
        directory. The original error is: \n""" + cpt.get_exception_message(e))
51
except Exception as e:
52
    raise e
53
from . import unique_name
Y
Yu Yang 已提交
54

55
__all__ = [
56 57 58 59
    'Program',
    'default_startup_program',
    'default_main_program',
    'program_guard',
60
    'name_scope',
61
]
Y
Yu Yang 已提交
62

Q
qiaolongfei 已提交
63 64 65 66
EMPTY_VAR_NAME = core.kEmptyVarName()
TEMP_VAR_NAME = core.kTempVarName()
GRAD_VAR_SUFFIX = core.kGradVarSuffix()
ZERO_VAR_SUFFIX = core.kZeroVarSuffix()
W
Wu Yi 已提交
67 68
CONTROL_DEP_VAR_PREFIX = core.kControlDepVarName()

69 70 71 72 73 74 75 76 77 78
_imperative_tracer_ = None


def _in_imperative_mode():
    return _imperative_tracer_ is not None


def _imperative_tracer():
    return _imperative_tracer_

W
Wu Yi 已提交
79

80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
class NameScope(object):
    def __init__(self, name="", parent=None):
        self._children = dict()
        self._name = name
        self._parent = parent

    def child(self, prefix):
        if prefix not in self._children:
            new_child = NameScope(prefix, self)
            self._children[prefix] = [new_child]
        else:
            new_child = NameScope(prefix + "_%d" % len(self._children[prefix]),
                                  self)
            self._children[prefix].append(new_child)
        return new_child

    def parent(self):
        return self._parent

    def name(self):
        return self._name


_name_scope = NameScope()


@contextlib.contextmanager
def name_scope(prefix=None):
    """
    Generate hierarchical name prefix for the operators.

    Note: This should only used for debugging and visualization purpose.
    Don't use it for serious analysis such as graph/program transformations.

    Args:
        prefix(str): prefix.

    Examples:
        .. code-block:: python
T
Tink_Y 已提交
119

120 121 122 123
          with name_scope("encoder"):
             ...
          with name_scope("decoder"):
             ...
T
Tink_Y 已提交
124 125
          with name_scope("attention"):
             ...
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
    """
    # TODO(panyx0718): Only [0-9a-z].
    assert prefix, "namescope prefix cannot be empty."
    global _name_scope
    _name_scope = _name_scope.child(prefix)
    yield
    _name_scope = _name_scope.parent()


def _full_name_scope():
    global _name_scope
    scope = _name_scope
    name = ""
    while scope:
        name = scope.name() + "/" + name
        scope = scope.parent()
    return name


W
Wu Yi 已提交
145 146 147
def generate_control_dev_var_name():
    import random
    return CONTROL_DEP_VAR_PREFIX + "@" + str(random.random())
Q
qiaolongfei 已提交
148 149 150 151


def grad_var_name(var_name):
    """
152 153
    Returns:
        str: gradient name for a certain var name
Q
qiaolongfei 已提交
154 155 156
    """
    return var_name + GRAD_VAR_SUFFIX

Y
Yu Yang 已提交
157

158
def convert_np_dtype_to_dtype_(np_dtype):
159 160
    """
    Convert the data type in numpy to the data type in Paddle
161

162
    Args:
163
        np_dtype(np.dtype): the data type in numpy.
164

165 166
    Returns:
        core.VarDesc.VarType: the data type in Paddle.
167 168

    """
169 170
    dtype = np.dtype(np_dtype)
    if dtype == np.float32:
171
        return core.VarDesc.VarType.FP32
172
    elif dtype == np.float64:
173
        return core.VarDesc.VarType.FP64
174
    elif dtype == np.float16:
175
        return core.VarDesc.VarType.FP16
176
    elif dtype == np.int32:
177
        return core.VarDesc.VarType.INT32
178
    elif dtype == np.int16:
179
        return core.VarDesc.VarType.INT16
180
    elif dtype == np.int64:
181
        return core.VarDesc.VarType.INT64
182
    elif dtype == np.bool:
183
        return core.VarDesc.VarType.BOOL
184 185
    elif dtype == np.uint16:
        return core.VarDesc.VarType.INT16
186 187
    elif dtype == np.uint8:
        return core.VarDesc.VarType.UINT8
Q
qingqing01 已提交
188 189
    elif dtype == np.int8:
        return core.VarDesc.VarType.INT8
190
    else:
M
minqiyang 已提交
191
        raise ValueError("Not supported numpy dtype %s" % dtype)
192 193 194


def dtype_is_floating(dtype):
195 196 197
    """
    Check the data type is floating or not.
    Args:
198
        dtype(np.dtype|core.VarDesc.VarType): data type.
199 200 201 202 203
            Could be numpy format or Paddle format

    Returns(bool): True if data type is a float value

    """
204
    if not isinstance(dtype, core.VarDesc.VarType):
205 206
        dtype = convert_np_dtype_to_dtype_(dtype)

207 208 209 210
    return dtype in [
        core.VarDesc.VarType.FP16, core.VarDesc.VarType.FP32,
        core.VarDesc.VarType.FP64
    ]
211 212


Y
Yang Yang(Tony) 已提交
213
def _debug_string_(proto, throw_on_error=True):
214 215 216 217 218 219 220 221 222 223 224
    """
    Get the debug string of a protobuf message. The message could be not
    initialized.
    Args:
        proto(google.protobuf.message.Message): The protobuf message
        throw_on_error(bool): True if raise an error when the protobuf message
            is not initialized.

    Returns(str): The debug string of the protobuf message

    """
Y
Yu Yang 已提交
225
    error_fields = list()
Y
Yang Yang(Tony) 已提交
226
    if not proto.IsInitialized(error_fields) and throw_on_error:
C
caoying03 已提交
227 228
        raise ValueError("{0} are not initialized.\nThe message is {1}:\n".
                         format(error_fields, proto))
Y
Yu Yang 已提交
229 230 231
    return proto.__str__()


X
Xin Pan 已提交
232
class Variable(object):
233
    """
234 235 236
    In Fluid, every input and output of an operator is a variable. In most
    cases, variables are used for holding different kinds of data or training
    labels. A variable belongs to a block. All variable has its own name and
237
    two variables in different blocks could have the same name.
238

239 240
    There are many kinds of variables. Each kind of them has its own attributes
    and usages. Please reference the framework.proto for details.
241

242
    Most of a Variable's member variables can be setted to be None. It mean
243
    it is not available or will be specified later.
244 245

    Args:
246
        block(Block): The block that the variable belongs to.
247 248
        type(core.VarDesc.VarType): Variable type. Please reference the
            framework.proto for details.
249 250
        name(str|None): The name of the variable. If setted None, it will be
            generated automatically. Default: None
251
        shape(tuple|list|None): The shape of the variable. -1 means the batch size.
252
            Some kinds of variable do not contain shape, just set it to None.
253 254 255
            Default: None
        dtype(np.dtype|core.VarDesc.VarType|str|None): The data type of variable.
            Default: None
256
        lod_level (int|None): The level of lod tensor. 0 means it is not a time
257
            series data.
258
            Default: None
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
        capacity (int|None): The capacity of Channel variable. Ignored for other
            types. Default: None
        persistable (bool|None): True if the variable is persistable. A persistable
            variable will not be deleted after an iteration ending. Defaults: None.
        error_clip (BaseErrorClipAttr|None): The error clip attributes of the
            corresponding gradient variable. Default: None
        stop_gradient (bool): True if the variable will stop to calculate its
            gradients when backward. Default: False.
        is_data (bool): True if the variable is an input data. Default: False

    Notes:
        The constructor of Variable should not be invoked directly. Please
        use `Block.create_var` to create a variable.

    Examples:
        .. code-block:: python

            cur_program = Program()
            cur_block = cur_program.current_block()
            new_variable = cur_block.create_var(name="X",
                                                shape=[-1, 23, 48],
                                                dtype='float32')
281 282
    """

Y
Yu Yang 已提交
283 284
    def __init__(self,
                 block,
Y
Yu Yang 已提交
285
                 type=core.VarDesc.VarType.LOD_TENSOR,
Y
Yu Yang 已提交
286 287 288 289
                 name=None,
                 shape=None,
                 dtype=None,
                 lod_level=None,
290
                 capacity=None,
Q
QI JUN 已提交
291
                 persistable=None,
F
fengjiayi 已提交
292
                 error_clip=None,
Y
Yu Yang 已提交
293
                 stop_gradient=False,
F
fengjiayi 已提交
294
                 is_data=False,
Y
Yu Yang 已提交
295
                 **kwargs):
Y
Yu Yang 已提交
296
        self.block = block
F
fengjiayi 已提交
297
        self.error_clip = error_clip
Y
Yu Yang 已提交
298 299

        if name is None:
Y
Yu Yang 已提交
300
            name = unique_name.generate('_generated_var')
D
Dong Zhihong 已提交
301
        is_new_var = False
M
minqiyang 已提交
302
        name = cpt.to_text(name)
M
minqiyang 已提交
303
        self.desc = self.block.desc.find_var(cpt.to_bytes(name))
D
Dong Zhihong 已提交
304 305

        if self.desc is None:
M
minqiyang 已提交
306
            self.desc = self.block.desc.var(cpt.to_bytes(name))
Y
Yu Yang 已提交
307
            is_new_var = True
Y
Yu Yang 已提交
308

Y
Yu Yang 已提交
309 310 311 312 313 314 315 316
        if is_new_var:
            self.desc.set_type(type)
        elif self.desc.type() != type:
            raise ValueError("Variable {0} has been created before. The "
                             "previous type is {1}; the new type is {2}. They"
                             " are not matched".format(self.name,
                                                       self.desc.type(), type))

Y
Yu Yang 已提交
317
        if shape is not None:
Y
Yu Yang 已提交
318
            if is_new_var:
319
                self.desc.set_shape(shape)
Y
Yu Yang 已提交
320 321 322 323 324 325 326 327
            else:
                old_shape = self.shape
                shape = tuple(shape)
                if shape != old_shape:
                    raise ValueError(
                        "Variable {0} has been created before. the previous "
                        "shape is {1}; the new shape is {2}. They are not "
                        "matched.".format(self.name, old_shape, shape))
Y
Yu Yang 已提交
328
        if dtype is not None:
329
            if not isinstance(dtype, core.VarDesc.VarType):
330
                dtype = convert_np_dtype_to_dtype_(dtype)
Y
Yu Yang 已提交
331
            if is_new_var:
F
fengjiayi 已提交
332
                self.desc.set_dtype(dtype)
Y
Yu Yang 已提交
333
            else:
F
fengjiayi 已提交
334
                old_dtype = self.dtype
Q
QI JUN 已提交
335
                if dtype != old_dtype:
Y
Yu Yang 已提交
336 337 338 339 340
                    raise ValueError("Variable {0} has been created before. "
                                     "The previous data type is {1}; the new "
                                     "data type is {2}. They are not "
                                     "matched.".format(self.name, old_dtype,
                                                       dtype))
Y
Yu Yang 已提交
341 342

        if lod_level is not None:
Y
Yu Yang 已提交
343
            if is_new_var:
344
                self.desc.set_lod_level(lod_level)
Y
Yu Yang 已提交
345 346 347 348 349 350 351
            else:
                if lod_level != self.lod_level:
                    raise ValueError("Variable {0} has been created before. "
                                     "The previous lod_level is {1}; the new "
                                     "lod_level is {2}. They are not "
                                     "matched".format(self.name, self.lod_level,
                                                      lod_level))
352 353 354 355 356 357 358 359 360 361 362
        if persistable is not None:
            if is_new_var:
                self.desc.set_persistable(persistable)
            else:
                if persistable != self.persistable:
                    raise ValueError(
                        "Variable {0} has been created before."
                        "The previous persistable is {1}; the new "
                        "persistable is {2}. They are not matched".format(
                            self.name, self.persistable, persistable))

363 364 365 366 367 368 369 370
        if capacity is not None:
            if is_new_var:
                self.desc.set_capacity(capacity)
            else:
                # TODO(abhinavarora) : Compare with set capacity once,
                # get_capacity is implemented
                pass

Y
Yu Yang 已提交
371
        self.block.vars[name] = self
Y
Yu Yang 已提交
372
        self.op = None
M
minqiyang 已提交
373
        self.stop_gradient = stop_gradient
F
fengjiayi 已提交
374
        self.is_data = is_data
X
Xin Pan 已提交
375
        if _in_imperative_mode():
M
minqiyang 已提交
376 377 378
            self._ivar = kwargs.get("ivar", None)
            if not self._ivar:
                self._ivar = core.VarBase()
X
Xin Pan 已提交
379
            self._ivar.desc = self.desc
380
            self._ivar.stop_gradient = stop_gradient
Y
Yu Yang 已提交
381

382
    def _numpy(self):
M
minqiyang 已提交
383
        tensor = self._ivar.value().get_tensor()
384 385 386
        return np.array(tensor)

    def _backward(self):
X
Xin Pan 已提交
387
        self._ivar._run_backward()
388 389

    def _gradient(self):
M
minqiyang 已提交
390
        return np.array(self._ivar._grad_value())
M
minqiyang 已提交
391

392
    def __str__(self):
Y
Yang Yang(Tony) 已提交
393 394
        return self.to_string(True)

F
update  
fengjiayi 已提交
395
    def to_string(self, throw_on_error, with_details=False):
396 397 398 399
        """
        Get debug string.

        Args:
400 401
            throw_on_error(bool): True if raise an exception when self is
                not initialized.
F
update  
fengjiayi 已提交
402
            with_details(bool): more details about variables and parameters
403 404
                (e.g. trainable, optimize_attr, ...) will be printed when
                with_details is True. Default False;
405

406 407
        Returns:
            str: The debug string.
408
        """
F
update  
fengjiayi 已提交
409 410
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
411
        protostr = self.desc.serialize_to_string()
412
        proto = framework_pb2.VarDesc.FromString(six.binary_type(protostr))
F
update  
fengjiayi 已提交
413 414 415 416
        res_str = _debug_string_(proto, throw_on_error)
        if with_details:
            additional_attr = ("error_clip", "stop_gradient")
            for attr_name in additional_attr:
417 418
                res_str += "%s: %s\n" % (
                    attr_name, six.binary_type(getattr(self, attr_name)))
F
update  
fengjiayi 已提交
419
        return res_str
420 421 422

    __repr__ = __str__

W
Wu Yi 已提交
423
    def _set_desc(self, input):
424 425 426 427 428 429 430 431 432
        """
        Set the variable description.

        Args:
            input(core.VarDesc): The new VarDesc.

        Returns:
            None
        """
433 434
        self.desc = input

435 436 437 438 439 440 441 442
    @property
    def _stop_gradient(self):
        return self._ivar.stop_gradient

    @_stop_gradient.setter
    def _stop_gradient(self, s):
        self._ivar.stop_gradient = s

443 444 445 446
    @property
    def persistable(self):
        return self.desc.persistable()

Y
Yu Yang 已提交
447 448 449 450
    @persistable.setter
    def persistable(self, p):
        self.desc.set_persistable(p)

Y
Yu Yang 已提交
451 452
    @property
    def name(self):
M
minqiyang 已提交
453
        return cpt.to_text(self.desc.name())
Y
Yu Yang 已提交
454

T
typhoonzero 已提交
455 456 457 458
    @name.setter
    def name(self, new_name):
        self.desc.set_name(new_name)

Y
Yu Yang 已提交
459 460 461
    @property
    def shape(self):
        # convert to tuple, make it as same as numpy API.
462
        return tuple(self.desc.shape())
Y
Yu Yang 已提交
463 464

    @property
F
fengjiayi 已提交
465 466
    def dtype(self):
        return self.desc.dtype()
Y
Yu Yang 已提交
467 468 469

    @property
    def lod_level(self):
470
        return self.desc.lod_level()
Y
Yu Yang 已提交
471

Y
Yu Yang 已提交
472 473 474 475
    @property
    def type(self):
        return self.desc.type()

W
Wu Yi 已提交
476
    def _set_error_clip(self, error_clip):
477 478 479 480 481 482 483 484 485
        """
        Set the error_clip.

        Args:
            error_clip(BaseErrorClipAttr) : The new error_clip.

        Returns:
            None
        """
486 487
        self.error_clip = error_clip

Y
Yu Yang 已提交
488

F
fengjiayi 已提交
489 490 491
def get_all_op_protos():
    """
    Get all registered op proto from PaddlePaddle C++ end.
492

493 494
    Returns:
       list: list of OpProto.
F
fengjiayi 已提交
495 496 497 498
    """
    protostrs = core.get_all_op_protos()
    ret_values = []
    for pbstr in protostrs:
499
        op_proto = framework_pb2.OpProto.FromString(six.binary_type(pbstr))
F
fengjiayi 已提交
500 501 502 503 504
        ret_values.append(op_proto)
    return ret_values


class OpProtoHolder(object):
505 506 507 508
    """
    A global variable to hold all OpProtos from C++ as a map
    """

F
fengjiayi 已提交
509 510 511 512 513 514 515 516 517
    @classmethod
    def instance(cls):
        if not hasattr(cls, '_instance'):
            cls._instance = cls()
        return cls._instance

    def __init__(self):
        assert not hasattr(
            self.__class__,
518
            '_instance'), 'Please use `instance()` to get OpProtoHolder object!'
F
fengjiayi 已提交
519 520 521 522 523 524
        op_protos = get_all_op_protos()
        self.op_proto_map = {}
        for proto in op_protos:
            self.op_proto_map[proto.type] = proto

    def get_op_proto(self, type):
525 526 527 528 529 530 531 532
        """
        Get OpProto by a type string.
        Args:
            type(str): The type that operator registered in C++ side.

        Returns(framework_pb2.OpProto): The OpProto

        """
Y
Yu Yang 已提交
533 534
        if type not in self.op_proto_map:
            raise ValueError("Operator \"%s\" has not been registered." % type)
F
fengjiayi 已提交
535 536
        return self.op_proto_map[type]

537 538 539 540
    @staticmethod
    def generated_op_attr_names():
        return {
            core.op_proto_and_checker_maker.kOpRoleAttrName(),
S
sneaxiy 已提交
541
            core.op_proto_and_checker_maker.kOpRoleVarAttrName(),
542
            core.op_proto_and_checker_maker.kOpNameScopeAttrName()
543 544
        }

F
fengjiayi 已提交
545

X
Xin Pan 已提交
546
class Operator(object):
547
    """
548 549 550 551 552 553 554
    In Fluid, all the operation are represented by Operator, and Operator
    is regarded as a build in an instruction of a Block. Users can use the
    build in instructions to describe their neural network.

    Args:
        block(Block): The block has the current operator.
        desc(core.OpDesc): The protobuf description of Operator.
C
chengduoZH 已提交
555
        type(str): The type of operator. Default None.
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575
        inputs(dict): The input of this Operator. it is a dictionary, for every
            element, key is the input parameter name, and value is a list of
            variables. Default None.
        outputs(dict): The output of this Operator. it is a dictionary, for
            every element, key is the input parameter name, and value is a list
            of variables. Default None.
        attrs(dict): The attributes of this Operator. it is a dictionary, for
            every element, key is attribute name, and value is the attribute value.
            The attribute type should be as same as the type registered in C++ side.
            Default None.

    Returns:
        Operator: The initialized Operator.

    Raises:
        ValueError: If the passed input, output and attrs doesn't match the
            initializing Operator's that registered in C++ side.

    Notes:
        The constructor of operator should not be invoked directly. Use
W
Wu Yi 已提交
576
        Block.append_op or Block._prepend_op instead.
577 578 579 580 581 582 583 584 585 586

    Examples:
        .. code-block:: python

            cur_program = Program()
            cur_block = cur_program.current_block()
            # var1 += var2 + var3
            cur_block.append_op(type="sum",
                                inputs={"X": [var1, var2, var3]},
                                outputs={"Out": [var1]})
587
    """
588 589 590
    OP_WITHOUT_KERNEL_SET = {
        'feed', 'fetch', 'save', 'load', 'recurrent', 'go',
        'rnn_memory_helper_grad', 'conditional_block', 'while', 'send', 'recv',
X
Xin Pan 已提交
591 592
        'listen_and_serv', 'save_combine', 'load_combine', 'ncclInit', 'select',
        'checkpoint_notify', 'gen_nccl_id'
593
    }
594

Y
Yu Yang 已提交
595 596
    def __init__(self,
                 block,
Y
Yu Yang 已提交
597
                 desc,
Y
Yu Yang 已提交
598 599 600
                 type=None,
                 inputs=None,
                 outputs=None,
M
minqiyang 已提交
601
                 attrs=None):
Y
Yu Yang 已提交
602
        self.block = block
Y
Yu Yang 已提交
603
        self.desc = desc
G
gongweibao 已提交
604 605 606 607 608
        # note: not add self.attrs here:
        # https://github.com/PaddlePaddle/Paddle/pull/12583#pullrequestreview-145093173
        op_attrs = attrs
        if op_attrs is None:
            op_attrs = dict()
Y
yuyang18 已提交
609 610 611 612
        del attrs

        op_maker = core.op_proto_and_checker_maker

G
gongweibao 已提交
613 614
        if op_maker.kOpRoleAttrName() not in op_attrs:
            op_attrs[op_maker.kOpRoleAttrName()] = self.block.program.op_role
Y
yuyang18 已提交
615 616 617

        role_var_name = op_maker.kOpRoleVarAttrName()
        if len(self.block.program.
G
gongweibao 已提交
618 619
               op_role_var) != 0 and role_var_name not in op_attrs:
            op_attrs[role_var_name] = self.block.program.op_role_var
Y
yuyang18 已提交
620

G
gongweibao 已提交
621 622
        if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0:
            del op_attrs[role_var_name]
Y
yuyang18 已提交
623

P
peizhilin 已提交
624 625 626 627 628 629 630 631 632
        if 'FLAGS_enable_debug' in os.environ and os.environ[
                'FLAGS_enable_debug']:
            callstack_var_name = op_maker.kOpCreationCallstackAttrName()
            op_attrs[callstack_var_name] = list(
                reversed(traceback.format_stack()))[1:]
            op_attrs[callstack_var_name].insert(
                0,
                'Invoke operator ' + (''
                                      if type is None else type) + ' error.\n')
633

F
fengjiayi 已提交
634 635 636 637 638
        if len(self.desc.type()) != 0:
            return
        if type is None:
            raise ValueError(
                "`type` to initilized an Operator can not be None.")
F
Update  
fengjiayi 已提交
639
        self.desc.set_type(type)
F
fengjiayi 已提交
640
        proto = OpProtoHolder.instance().get_op_proto(type)
641

642 643 644
        namescope_var_name = op_maker.kOpNameScopeAttrName()
        op_attrs[namescope_var_name] = _full_name_scope()

Y
Yang Yang(Tony) 已提交
645 646
        def find_name(var_list, name):
            for var_name in var_list:
Q
Qiao Longfei 已提交
647
                if var_list[var_name] is not None and var_name == name:
Y
Yang Yang(Tony) 已提交
648 649
                    return True
            return False
Q
QI JUN 已提交
650

Y
Yang Yang(Tony) 已提交
651 652 653 654 655 656 657
        if inputs is not None:
            for in_proto in proto.inputs:
                found = find_name(inputs, in_proto.name)
                assert found or in_proto.dispensable, "Input {} not found".format(
                    in_proto.name)

                if found:
658 659 660 661
                    in_args = inputs[in_proto.name]
                    if not isinstance(in_args, list):
                        in_args = [in_args]
                    if not in_proto.duplicable and len(in_args) > 1:
Y
Yang Yang(Tony) 已提交
662 663
                        raise ValueError(
                            "Input %s expects only one input, but %d are given."
664 665 666
                            % (in_proto.name, len(in_args)))
                    in_arg_names = []
                    for arg in in_args:
667
                        if isinstance(arg, six.string_types):
Y
Yang Yu 已提交
668
                            in_arg_names.append(arg)
669 670
                        elif isinstance(arg, six.binary_type):
                            in_arg_names.append(arg.decode())
Y
Yang Yu 已提交
671
                        else:
M
minqiyang 已提交
672
                            in_arg_names.append(cpt.to_text(arg.name))
673
                    self.desc.set_input(in_proto.name, in_arg_names)
Y
Yang Yang(Tony) 已提交
674 675
                else:
                    self.desc.set_input(in_proto.name, [])
F
Update  
fengjiayi 已提交
676

Y
Yu Yang 已提交
677
        if outputs is not None:
678
            for m in proto.outputs:
Q
qingqing01 已提交
679 680 681 682 683 684
                if (m.name not in outputs) and m.dispensable:
                    continue
                if not ((m.name in outputs) or m.dispensable):
                    raise ValueError(
                        ("Incorrect setting for output(s) of "
                         "operator \"%s\", should set: [%s].") % (type, m.name))
F
fengjiayi 已提交
685
            for out_proto in proto.outputs:
Q
qingqing01 已提交
686 687
                if out_proto.name not in outputs:
                    continue
688 689 690 691
                out_args = outputs[out_proto.name]
                if not isinstance(out_args, list):
                    out_args = [out_args]
                if not out_proto.duplicable and len(out_args) > 1:
F
Update  
fengjiayi 已提交
692 693
                    raise ValueError(
                        "Output %s expects only one output, but %d are given." %
694 695 696
                        (out_proto.name, len(out_args)))
                out_arg_names = []
                for arg in out_args:
M
minqiyang 已提交
697
                    out_arg_names.append(cpt.to_text(arg.name))
698 699
                    arg.op = self
                self.desc.set_output(out_proto.name, out_arg_names)
F
Update  
fengjiayi 已提交
700

G
gongweibao 已提交
701 702
        if op_attrs is not None:
            if not isinstance(op_attrs, dict):
703
                raise TypeError("'attrs' should be a dict.")
F
fengjiayi 已提交
704
            for attr in proto.attrs:
F
Update  
fengjiayi 已提交
705
                attr_name = attr.name
G
gongweibao 已提交
706
                if (attr_name not in op_attrs) or (op_attrs[attr_name] is None):
F
Update  
fengjiayi 已提交
707
                    continue
G
gongweibao 已提交
708
                attr_val = op_attrs[attr_name]
G
gongweibao 已提交
709 710
                self._update_desc_attr(attr_name, attr_val)

711
        self.desc.check_attrs()
M
minqiyang 已提交
712

W
Wu Yi 已提交
713
        if self._has_kernel(type):
Q
QI JUN 已提交
714
            self.desc.infer_var_type(self.block.desc)
Y
Yu Yang 已提交
715
            self.desc.infer_shape(self.block.desc)
M
minqiyang 已提交
716

X
Xin Pan 已提交
717 718 719
        if _in_imperative_mode():
            self.iop = core.OpBase()
            self.iop.desc = self.desc
X
Xin Pan 已提交
720
            self.inputs = defaultdict(list)
X
Xin Pan 已提交
721
            if inputs is not None:
X
Xin Pan 已提交
722 723 724 725 726 727
                for k, v in six.iteritems(inputs):
                    if isinstance(v, Variable):
                        self.inputs[k].append(v._ivar)
                    elif isinstance(v, list) or isinstance(v, tuple):
                        self.inputs[k].extend([var._ivar for var in v])
            self.outputs = defaultdict(list)
X
Xin Pan 已提交
728
            if outputs is not None:
X
Xin Pan 已提交
729 730 731 732 733
                for k, v in six.iteritems(outputs):
                    if isinstance(v, Variable):
                        self.outputs[k].append(v._ivar)
                    elif isinstance(v, list) or isinstance(v, tuple):
                        self.outputs[k].extend([var._ivar for var in v])
F
fengjiayi 已提交
734

W
Wu Yi 已提交
735
    def _has_kernel(self, op_type):
736 737
        return op_type not in self.OP_WITHOUT_KERNEL_SET

Y
Yang Yang(Tony) 已提交
738
    def to_string(self, throw_on_error):
739
        """
740 741
        Get debug string.

742
        Args:
743 744
            throw_on_error(bool): Whether to raise exception if self is not
                initialized.
745

746 747
        Returns:
            str: The debug string.
748 749

        """
750
        protostr = self.desc.serialize_to_string()
751
        proto = framework_pb2.OpDesc.FromString(six.binary_type(protostr))
Y
Yang Yang(Tony) 已提交
752 753 754 755
        return _debug_string_(proto, throw_on_error)

    def __str__(self):
        return self.to_string(True)
756 757 758

    __repr__ = __str__

F
fengjiayi 已提交
759 760 761 762 763
    @property
    def type(self):
        return self.desc.type()

    def input(self, name):
764
        """
765
        Get the input arguments according to the input parameter name.
766

767 768
        Args:
            name(str): The input parameter name.
769

770 771 772
        Returns:
            list: return the list of argument names that associated with \
                the specific parameter name.
773
        """
F
fengjiayi 已提交
774 775
        return self.desc.input(name)

W
Wu Yi 已提交
776
    def _rename_input(self, old_name, new_name):
777 778 779 780 781 782 783 784 785 786
        """
        Rename the `old_name` to `new_name`.

        Args:
            old_name(str): The old name of the Operator's input.
            new_name(str): The new name of the Operator's input.

        Returns:
            None
        """
W
Wu Yi 已提交
787
        self.desc._rename_input(old_name, new_name)
T
typhoonzero 已提交
788

W
Wu Yi 已提交
789
    def _rename_output(self, old_name, new_name):
790 791 792 793 794 795 796 797 798 799
        """
        Rename the `old_name` to `new_name`.

        Args:
            old_name(str): The old name of the Operator's output.
            new_name(str): The new name of the Operator's output.

        Returns:
            None
        """
W
Wu Yi 已提交
800
        self.desc._rename_output(old_name, new_name)
T
typhoonzero 已提交
801

F
fengjiayi 已提交
802 803 804 805
    @property
    def input_names(self):
        return self.desc.input_names()

T
typhoonzero 已提交
806 807 808 809 810 811 812 813
    @property
    def input_arg_names(self):
        return self.desc.input_arg_names()

    @property
    def output_arg_names(self):
        return self.desc.output_arg_names()

F
fengjiayi 已提交
814
    def output(self, name):
815
        """
816
        Get output arguments by the output parameter name.
817

818 819
        Args:
            name(str): The output parameter name.
820

821 822 823
        Returns:
            list: return the list of argument names associated with \
                the specific parameter name.
824
        """
F
fengjiayi 已提交
825 826 827 828 829 830
        return self.desc.output(name)

    @property
    def output_names(self):
        return self.desc.output_names()

831 832 833 834 835 836 837 838
    @property
    def idx(self):
        for i, op in enumerate(self.block.ops):
            if op == self:
                return i
        raise ValueError(
            "Can't find op itself in it's block. It could be a bug of Paddle.")

F
fengjiayi 已提交
839
    def has_attr(self, name):
840
        """
841 842
        Whether this Operator has the attribute with name or not.

843
        Args:
844
            name(str): the attribute name.
845

846 847
        Returns:
            bool: True if has this attribute.
848 849

        """
F
fengjiayi 已提交
850 851 852
        return self.desc.has_attr(name)

    def attr_type(self, name):
853
        """
854
        Get the type of attribute by attribute's name.
855

856 857
        Args:
            name(str): the attribute name.
858

859 860
        Returns:
            core.AttrType: the attribute type.
861
        """
F
fengjiayi 已提交
862 863
        return self.desc.attr_type(name)

W
Wu Yi 已提交
864
    def _set_attr(self, name, val):
865 866 867 868 869 870 871 872 873 874
        """
        Set the value of attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.

        Raises:
            ValueError: If the type of value doesn't match with desc.attr_type(name).
        """
G
gongweibao 已提交
875 876 877 878 879 880 881 882 883 884 885 886 887
        self._update_desc_attr(name, val)

    def _update_desc_attr(self, name, val):
        """
        Update the value of desc's attribute by attribute's name.

        Args:
            name(str): the attribute name.
            val(bool|int|str|float|list): the value of the attribute.

        Raises:
            ValueError: If the type of value doesn't match with desc.attr_type(name).
        """
Q
Qiyang Min 已提交
888 889
        if isinstance(val, Block):
            self.desc.set_block_attr(name, val.desc)
Y
Yancey1989 已提交
890 891
        elif isinstance(val, list) and val and all(
                isinstance(v, Block) for v in val):
892
            self.desc.set_blocks_attr(name, [v.desc for v in val])
Q
Qiyang Min 已提交
893 894 895 896
        elif isinstance(val, core.BlockDesc) or \
                isinstance(val, core.ProgramDesc):
            self.desc.set_serialized_attr(name, val.serialize_to_string())
        else:
W
Wu Yi 已提交
897
            self.desc._set_attr(name, val)
Y
yuyang18 已提交
898

F
fengjiayi 已提交
899 900 901 902 903
    @property
    def attr_names(self):
        return self.desc.attr_names()

    def attr(self, name):
904
        """
905 906
        Get the attribute by name.

907
        Args:
908
            name(str): the attribute name.
909

910 911
        Returns:
            bool|int|str|float|list: The attribute value. The return value
912 913
            can be any valid attribute type.
        """
F
fengjiayi 已提交
914
        return self.desc.attr(name)
Y
Yu Yang 已提交
915

W
Wu Yi 已提交
916
    def _block_attr_id(self, name):
917
        """
G
gongweibao 已提交
918
        Get the block attribute's id by name.
919

920 921
        Args:
            name(str): the attribute name.
922

923 924
        Returns:
            int: the block index.
925
        """
W
Wu Yi 已提交
926
        return self.desc._block_attr_id(name)
G
gongweibao 已提交
927

W
Wu Yi 已提交
928
    def _block_attr(self, name):
G
gongweibao 已提交
929 930 931 932 933 934 935 936 937 938
        """
        Get the block attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            block: the block attribute.
        """

W
Wu Yi 已提交
939
        id = self._block_attr_id(name)
G
gongweibao 已提交
940 941 942
        assert (id >= 0 and id < len(self.block.program.blocks))
        return self.block.program.blocks[id]

W
Wu Yi 已提交
943
    def _blocks_attr(self, name):
G
gongweibao 已提交
944 945 946 947 948 949 950 951 952 953
        """
        Get the blocks attribute  by name.

        Args:
            name(str): the attribute name.

        Returns:
            list: list of the blocks attribute.
        """
        attrs = []
W
Wu Yi 已提交
954
        for i in self._blocks_attr_ids(name):
G
gongweibao 已提交
955 956 957 958 959
            assert (i >= 0 and i < len(self.block.program.blocks))
            attrs.append(self.block.program.blocks[i])

        return attrs

W
Wu Yi 已提交
960
    def _blocks_attr_ids(self, name):
G
gongweibao 已提交
961 962 963 964 965 966 967 968 969 970
        """
        Get the blocks attribute's ids by name.

        Args:
            name(str): the attribute name.

        Returns:
            list: list of the blocks ids.
        """

W
Wu Yi 已提交
971
        return self.desc._blocks_attr_ids(name)
Y
Yu Yang 已提交
972

J
JiayiFeng 已提交
973
    def all_attrs(self):
F
fengjiayi 已提交
974
        """
975 976 977
        Get the attribute dict.

        Returns:
G
gongweibao 已提交
978
            dict: The Operator's attribute dict, name->attr.
F
fengjiayi 已提交
979 980 981 982
        """
        attr_names = self.attr_names
        attr_map = {}
        for n in attr_names:
G
gongweibao 已提交
983 984
            attr_type = self.desc.attr_type(n)
            if attr_type == core.AttrType.BLOCK:
W
Wu Yi 已提交
985
                attr_map[n] = self._block_attr(n)
G
gongweibao 已提交
986 987 988
                continue

            if attr_type == core.AttrType.BLOCKS:
W
Wu Yi 已提交
989
                attr_map[n] = self._blocks_attr(n)
G
gongweibao 已提交
990 991 992 993
                continue

            attr_map[n] = self.attr(n)

F
fengjiayi 已提交
994 995
        return attr_map

Y
Yu Yang 已提交
996

Y
Yu Yang 已提交
997
class Block(object):
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
    """
    In Fluid, a Program is consistence of multi-Block, and Block stores
    VarDesc and OpDesc. In a specific Block, a VarDesc have a unique name.
    One block could have some child blocks, and child block's name scopes
    should inherit the parent's so that OpDesc in child block can reference
    a VarDesc that is stored in the parent block.
    Please reference the framework.proto for details.

    Args:
        program(Program): The Program that the Block belongs to.
        idx(int): The block's id in the Program.

    Notes:
        The constructor of Block should not be invoked directly. Please
W
Wu Yi 已提交
1012
        use `Program._create_block()` to create a block.
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026

    Examples:
        .. code-block:: python

            cur_program = Program()
            cur_block = cur_program.current_block()
            var = cur_block.create_var(name="X",
                                       shape=[-1, 23, 48],
                                       dtype='float32')
            cur_block.append_op(type="abs",
                                inputs={"X": [var]},
                                outputs={"Out": [var]})
    """

Y
Yu Yang 已提交
1027
    def __init__(self, program, idx):
Y
Yu Yang 已提交
1028
        self.desc = program.desc.block(idx)
1029
        self.vars = collections.OrderedDict()  # var_name --> var
Q
qiaolongfei 已提交
1030
        self.ops = list()  # operator list
Y
Yu Yang 已提交
1031
        self.program = program
1032
        self.removed_vars = collections.OrderedDict()
Y
Yu Yang 已提交
1033

1034
    def __str__(self):
Y
Yang Yang(Tony) 已提交
1035 1036
        return self.to_string(True)

F
fengjiayi 已提交
1037 1038
    def to_string(self, throw_on_error, with_details=False):
        """
1039 1040
        Get debug string.

F
fengjiayi 已提交
1041 1042
        Args:
            throw_on_error(bool): raise exception when self is not initialized
1043
                when throw_on_error is True.
F
update  
fengjiayi 已提交
1044
            with_details(bool): more details about variables and parameters
1045 1046
                (e.g. trainable, optimize_attr, ...) will be printed when
                with_details is True. Default False.
F
fengjiayi 已提交
1047

1048 1049
        Returns:
            str: The debug string.
F
fengjiayi 已提交
1050 1051 1052 1053
        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
F
fengjiayi 已提交
1054
            re_add_indent = re.compile(r"\n(.)")
F
fengjiayi 已提交
1055 1056
            res_str = "blocks {\n  idx: %d\n  parent_idx: %d" % (
                self.idx, self.parent_idx)
1057
            for var in list(self.vars.values()):
F
fengjiayi 已提交
1058
                res_str += "\n  vars {\n    %s  }" % re_add_indent.sub(
F
update  
fengjiayi 已提交
1059
                    r"\n    \1", var.to_string(throw_on_error, with_details))
F
fengjiayi 已提交
1060
            for op in self.ops:
F
fengjiayi 已提交
1061 1062
                res_str += "\n  ops {\n    %s  }" % re_add_indent.sub(
                    r"\n    \1", op.to_string(throw_on_error))
F
fengjiayi 已提交
1063 1064 1065
            res_str += "\n}"
        else:
            protostr = self.desc.serialize_to_string()
1066 1067
            proto = framework_pb2.BlockDesc.FromString(
                six.binary_type(protostr))
F
fengjiayi 已提交
1068 1069
            res_str = _debug_string_(proto, throw_on_error)
        return res_str
1070 1071 1072

    __repr__ = __str__

Y
Yu Yang 已提交
1073 1074
    @property
    def parent_idx(self):
Y
Yu Yang 已提交
1075
        return self.desc.parent
Y
Yu Yang 已提交
1076

Y
Yu Yang 已提交
1077 1078 1079 1080
    @property
    def forward_block_idx(self):
        return self.desc.get_forward_block_idx()

W
Wu Yi 已提交
1081
    def _set_forward_block_idx(self, idx):
1082 1083 1084 1085 1086 1087 1088 1089 1090
        """
        Set the forward block Idx.

        Args:
            idx(int): the block index.

        Returns:
            None
        """
W
Wu Yi 已提交
1091
        self.desc._set_forward_block_idx(idx)
Y
Yu Yang 已提交
1092

Y
Yu Yang 已提交
1093 1094
    @property
    def idx(self):
Y
Yu Yang 已提交
1095
        return self.desc.id
Y
Yu Yang 已提交
1096

Q
Qiao Longfei 已提交
1097
    def var(self, name):
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
        """
        Get a Variable by name from this block.

        Args:
            name(str): the Variable's name.

        Raises:
            ValueError: The If input's type is not str, or this block
                doesn't have a Variable with the giving name.

        Returns:
            Variable: the Variable with the giving name.
        """
1111
        if not isinstance(name, six.string_types):
M
minqiyang 已提交
1112 1113 1114
            raise TypeError(
                "var require string as parameter, but get %s instead." %
                (type(name)))
Y
Yu Yang 已提交
1115 1116
        v = self.vars.get(name, None)
        if v is None:
Q
Qiao Longfei 已提交
1117
            raise ValueError("var %s not in this block" % name)
Y
Yu Yang 已提交
1118
        return v
Q
Qiao Longfei 已提交
1119

X
Xin Pan 已提交
1120
    def _find_var_recursive(self, name):
1121 1122 1123 1124 1125 1126 1127
        """
        Get a Variable by name from this block recursively.

        Args:
            name(str): the Variable's name.

        Returns:
X
Xin Pan 已提交
1128
            Variable: the Variable with the giving name. Or None if not found.
1129
        """
Y
Yu Yang 已提交
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
        frontier = list()
        visited = set()

        frontier.append(self)

        prog = self.program

        while len(frontier) != 0:  # BFS
            cur = frontier[0]
            frontier = frontier[1:]

            if id(cur) in visited:
                continue

            if cur.has_var(name):
                return cur.var(name)

            if cur.parent_idx != -1:
                frontier.append(prog.block(cur.parent_idx))

            if cur.forward_block_idx != -1:
                frontier.append(prog.block(cur.forward_block_idx))

            visited.add(id(cur))
X
Xin Pan 已提交
1154
        return None
Y
Yu Yang 已提交
1155

X
Xin Pan 已提交
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
    def _var_recursive(self, name):
        """
        Get a Variable by name from this block recursively.

        Args:
            name(str): the Variable's name.

        Raises:
            ValueError: this block and this parent block doesn't
                have a Variable with the giving name.

        Returns:
            Variable: the Variable with the giving name.
        """
        var = self._find_var_recursive(name)
        if var:
            return var
        else:
            raise ValueError("Var {0} is not found recursively".format(name))
F
fengjiayi 已提交
1175

Q
Qiao Longfei 已提交
1176
    def all_parameters(self):
1177
        return list(self.iter_parameters())
1178

1179
    def iter_parameters(self):
M
minqiyang 已提交
1180
        return (item[1] for item in six.iteritems(self.vars)
1181
                if isinstance(item[1], Parameter))
Q
Qiao Longfei 已提交
1182

Y
Yu Yang 已提交
1183
    def create_var(self, *args, **kwargs):
1184
        var = Variable(block=self, *args, **kwargs)
1185 1186
        if 'initializer' in kwargs:
            kwargs['initializer'](var, self)
Q
Qiao Longfei 已提交
1187
        return var
Y
Yu Yang 已提交
1188

Q
Qiao Longfei 已提交
1189 1190 1191
    def has_var(self, name):
        return name in self.vars

W
Wu Yi 已提交
1192
    def _rename_var(self, name, new_name):
T
typhoonzero 已提交
1193 1194
        """
        Rename variable in vars and ops' inputs and outputs
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206

        Args:
            name(str): the name that need to be renamed.
            new_name(str): the name that need to rename to.

        Raises:
            ValueError: If this block doesn't have this the giving name,
                or the type of the var with the giving name is not Parameter
                or Variable.

        Returns:
            Variable: the Variable with the giving name.
T
typhoonzero 已提交
1207
        """
M
minqiyang 已提交
1208 1209
        name = cpt.to_text(name)
        new_name = cpt.to_text(new_name)
M
minqiyang 已提交
1210

T
typhoonzero 已提交
1211
        if not self.has_var(name):
1212
            raise ValueError("var %s is not in current block" % name)
T
wip  
typhoonzero 已提交
1213 1214
        v = self.var(name)
        if type(v) == Parameter:
T
typhoonzero 已提交
1215
            var_type = "Parameter"
T
wip  
typhoonzero 已提交
1216 1217 1218 1219 1220 1221 1222
            stop_gradient = v.stop_gradient
            trainable = v.trainable
            optimize_attr = v.optimize_attr
            regularizer = v.regularizer
            gradient_clip_attr = v.gradient_clip_attr
            error_clip = v.error_clip
        elif type(v) == Variable:
T
typhoonzero 已提交
1223
            var_type = "Variable"
T
wip  
typhoonzero 已提交
1224 1225 1226 1227
            error_clip = v.error_clip
            stop_gradient = v.stop_gradient
        else:
            raise ValueError("unsupported var type: %s", type(v))
T
typhoonzero 已提交
1228
        orig_var_type = v.type
M
minqiyang 已提交
1229
        self.desc._rename_var(cpt.to_bytes(name), cpt.to_bytes(new_name))
W
Wu Yi 已提交
1230
        # NOTE: v is destroyed by C++ after calling _rename_var.
M
minqiyang 已提交
1231
        d = self.desc.find_var(cpt.to_bytes(new_name))
T
typhoonzero 已提交
1232
        if var_type == "Parameter":
T
wip  
typhoonzero 已提交
1233 1234 1235 1236
            var = Parameter(
                self,
                d.shape(),
                d.dtype(),
T
typhoonzero 已提交
1237
                type=orig_var_type,
T
wip  
typhoonzero 已提交
1238 1239 1240 1241 1242 1243 1244
                name=new_name,
                stop_gradient=stop_gradient,
                trainable=trainable,
                optimize_attr=optimize_attr,
                regularizer=regularizer,
                gradient_clip_attr=gradient_clip_attr,
                error_clip=error_clip)
T
typhoonzero 已提交
1245
        elif var_type == "Variable":
T
wip  
typhoonzero 已提交
1246 1247
            var = Variable(
                self,
T
typhoonzero 已提交
1248
                type=orig_var_type,
T
wip  
typhoonzero 已提交
1249 1250 1251 1252
                name=new_name,
                error_clip=error_clip,
                stop_gradient=stop_gradient)

W
Wu Yi 已提交
1253
        # rename the python side, _sync_with_cpp will only add
T
wip  
typhoonzero 已提交
1254 1255 1256
        # new vars/ops to python side.
        self.vars[new_name] = var
        del self.vars[name]
W
Wu Yi 已提交
1257
        self._sync_with_cpp()
1258
        return var
T
typhoonzero 已提交
1259

W
Wu Yi 已提交
1260 1261
    def _remove_var(self, name):
        self._sync_with_cpp()
M
minqiyang 已提交
1262
        self.desc._remove_var(cpt.to_bytes(name))
1263 1264
        del self.vars[name]

Y
Yu Yang 已提交
1265 1266
    def create_parameter(self, *args, **kwargs):
        global_block = self.program.global_block()
Q
Qiao Longfei 已提交
1267
        param = Parameter(global_block, *args, **kwargs)
1268
        if 'initializer' in kwargs:
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288

            def _is_inited_by(block, var):
                init_ops = []
                for op in block.ops:
                    if var.name in op.output_arg_names:
                        init_ops.append(op)
                return init_ops

            initializer = kwargs['initializer']
            init_ops = _is_inited_by(global_block, param)
            init_ops_len = len(init_ops)
            if init_ops_len > 1:
                raise RuntimeError("param " + param.name +
                                   " is inited by multiple init ops " + str(
                                       init_ops))
            elif init_ops_len == 1:
                #TODO already inited, do nothing, should log a warning
                pass
            else:
                initializer(param, self)
Q
Qiao Longfei 已提交
1289
        return param
Y
Yu Yang 已提交
1290

Y
Yu Yang 已提交
1291
    def append_op(self, *args, **kwargs):
1292 1293 1294 1295 1296 1297
        """
        Appends a new Operator according to the giving arguments.

        Returns:
            Operator: the append Operator.
        """
Y
Yu Yang 已提交
1298
        op_desc = self.desc.append_op()
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
        op = Operator(
            block=self,
            desc=op_desc,
            type=kwargs.get("type", None),
            inputs=kwargs.get("inputs", None),
            outputs=kwargs.get("outputs", None),
            attrs=kwargs.get("attrs", None))
        self.ops.append(op)
        self._trace_op(op, kwargs.get("stop_gradient", False))
        return op

    def _trace_op(self, op, stop_gradient=False):
1311
        if _in_imperative_mode():
1312
            _imperative_tracer().trace(op.iop, op.inputs, op.outputs, self.desc,
1313
                                       stop_gradient)
Y
Yu Yang 已提交
1314

W
Wu Yi 已提交
1315
    def _insert_op(self, index, *args, **kwargs):
1316 1317 1318 1319 1320 1321 1322 1323 1324
        """
        Insert a Operator according to the giving arguments.

        Args:
            index(int): the place that the operator to insert.

        Returns:
            Operator: the insert Operator.
        """
W
Wu Yi 已提交
1325 1326
        self._sync_with_cpp()
        op_desc = self.desc._insert_op(index)
Q
qiaolongfei 已提交
1327 1328 1329 1330
        op = Operator(block=self, desc=op_desc, *args, **kwargs)
        self.ops.insert(index, op)
        return op

W
Wu Yi 已提交
1331
    def _remove_op(self, index):
1332 1333 1334 1335 1336 1337 1338 1339 1340
        """
        Remove the specific position operator.

        Args:
            index(int): the position that the operator to insert.

        Returns:
            None
        """
W
Wu Yi 已提交
1341 1342
        self._sync_with_cpp()
        self.desc._remove_op(index, index + 1)
1343 1344
        del self.ops[index]

W
Wu Yi 已提交
1345
    def _slice_ops(self, start, end):
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355
        """
        Return the Operator between start and end.

        Args:
            start(int): the start position.
            end(int): the end position.

        Returns:
            list: the Operators between start and end.
        """
Q
qiaolongfei 已提交
1356
        return self.ops[start:end]
Y
Yancey1989 已提交
1357

W
Wu Yi 已提交
1358 1359
    def _prepend_op(self, *args, **kwargs):
        op_desc = self.desc._prepend_op()
1360 1361 1362 1363 1364 1365 1366
        op = Operator(
            self,
            op_desc,
            type=kwargs.get("type", None),
            inputs=kwargs.get("inputs", None),
            outputs=kwargs.get("outputs", None),
            attrs=kwargs.get("attrs", None))
Q
qiaolongfei 已提交
1367
        self.ops.insert(0, op)
1368
        self._trace_op(op, kwargs.get("stop_gradient", False))
Y
Yu Yang 已提交
1369 1370
        return op

W
Wu Yi 已提交
1371
    def _sync_with_cpp(self):
1372
        """
1373 1374
        Sync from the desc on the c++ end. This method is used to synchronize
        the c++ desc instance generated by backward.
1375
        """
Q
Qiao Longfei 已提交
1376 1377 1378 1379 1380
        # sync variables from cpp
        for var in self.desc.all_vars():
            if not self.has_var(var.name()):
                self.create_var(name=var.name(), desc=var, type=var.type())

1381
        # sync variables removed from c++ end
1382
        for var in list(self.vars.keys()):
M
minqiyang 已提交
1383
            if not self.desc.find_var(cpt.to_bytes(var)):
1384 1385
                self.vars.pop(var)

Q
Qiao Longfei 已提交
1386
        # sync operators from cpp
1387 1388 1389 1390
        ops_in_cpp = []
        for op_idx in range(0, self.desc.op_size()):
            ops_in_cpp.append(self.desc.op(op_idx))

Y
Yu Yang 已提交
1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
        if len(self.ops) != 0:
            first_op_in_python = self.ops[0].desc
            last_op_in_python = self.ops[len(self.ops) - 1].desc
            start_index = None
            end_index = None
            for index in range(len(ops_in_cpp)):
                if first_op_in_python == ops_in_cpp[index]:
                    start_index = index
                if last_op_in_python == ops_in_cpp[index]:
                    end_index = index
            assert start_index is not None
            assert end_index is not None
            assert start_index <= end_index
        else:
            start_index = 0
            end_index = -1
Q
Qiao Longfei 已提交
1407 1408 1409 1410 1411

        # sync ops append to the head of cpp_ops
        for index in range((start_index - 1 - 1), -1, -1):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
Q
qiaolongfei 已提交
1412
            self.ops.insert(0, op)
Q
Qiao Longfei 已提交
1413 1414 1415 1416 1417 1418 1419

        # sync ops append to the end of cpp_ops
        for index in range((end_index + 1), len(ops_in_cpp)):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
            self.ops.append(op)

1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
        # sync ops removed from c++ end
        if end_index != -1 and end_index < len(self.ops):
            ops_in_cpp_index = 0
            ops_in_python_index = 0
            while ops_in_python_index < len(
                    self.ops) and ops_in_cpp_index < len(ops_in_cpp):
                if self.ops[ops_in_python_index].desc != ops_in_cpp[
                        ops_in_cpp_index]:
                    del self.ops[ops_in_python_index]
                else:
                    ops_in_cpp_index += 1
                    ops_in_python_index += 1

Q
Qiao Longfei 已提交
1433 1434 1435 1436
        assert len(self.ops) == len(ops_in_cpp)
        for index in range(len(self.ops)):
            assert self.ops[index].desc == ops_in_cpp[index]

W
Wu Yi 已提交
1437
    def _copy_param_info_from(self, other):
1438
        """
1439 1440
        Copy the information of parameters from the other block.

1441
        Args:
1442 1443 1444 1445 1446
            other(Block): the other block.

        Raises:
            ValueError: If type of input is not Block, or the `other` and this
                block is not in the same topology.
1447 1448 1449 1450 1451

        Returns:
            None
        """
        if not isinstance(other, Block):
W
Wu Yi 已提交
1452 1453
            raise TypeError(
                "_copy_param_info_from should be invoked with Block")
1454
        for p in other.iter_parameters():
1455 1456 1457
            assert isinstance(p, Parameter)
            v = self.vars.get(p.name, None)
            if v is None:
W
Wu Yi 已提交
1458
                raise ValueError("_copy_param_info_from should be invoked with "
1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
                                 "same topology")
            assert isinstance(v, Variable)
            new_p = Parameter(
                block=self,
                shape=v.shape,
                dtype=v.dtype,
                type=v.type,
                lod_level=v.lod_level,
                stop_gradient=p.stop_gradient,
                trainable=p.trainable,
                optimize_attr=p.optimize_attr,
                regularizer=p.regularizer,
F
fengjiayi 已提交
1471
                gradient_clip_attr=p.gradient_clip_attr,
F
fengjiayi 已提交
1472
                error_clip=p.error_clip,
1473 1474 1475
                name=v.name)
            self.vars[new_p.name] = new_p

W
Wu Yi 已提交
1476
    def _clone_variable(self, var):
1477 1478
        """
        Clone a variable into current block.
1479

1480 1481 1482 1483
        Args:
            var: the variable to be cloned.

        Returns:
1484
            Variable: the new  variable cloned from 'var' in current block.
1485 1486
        """
        assert isinstance(var, Variable)
T
update  
typhoonzero 已提交
1487 1488 1489 1490 1491
        ret_var = None
        # make STEP_SCOPES var can be safely cloned.
        if var.type == core.VarDesc.VarType.STEP_SCOPES:
            ret_var = self.create_var(
                name=var.name, persistable=var.persistable, type=var.type)
T
tangwei12 已提交
1492 1493
        elif var.type == core.VarDesc.VarType.RAW:
            ret_var = self.create_var(
T
tangwei12 已提交
1494
                name=var.name, persistable=var.persistable, type=var.type)
T
typhoonzero 已提交
1495 1496 1497 1498 1499 1500
        elif var.type == core.VarDesc.VarType.SELECTED_ROWS:
            ret_var = self.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
F
fengjiayi 已提交
1501 1502
                persistable=True,
                is_data=var.is_data)
T
update  
typhoonzero 已提交
1503 1504 1505 1506 1507 1508 1509
        else:
            ret_var = self.create_var(
                name=var.name,
                shape=var.shape,
                dtype=var.dtype,
                type=var.type,
                lod_level=var.lod_level,
F
fengjiayi 已提交
1510 1511
                persistable=True,
                is_data=var.is_data)
T
update  
typhoonzero 已提交
1512
        return ret_var
1513

Y
Yu Yang 已提交
1514 1515

class Program(object):
D
dzhwinter 已提交
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526
    """
    Python Program. Beneath it is a ProgramDesc, which is used for
    create c++ Program. A program is a self-contained programing
    language like container. It has at least one Block, when the
    control flow op like conditional_block, while_op is included,
    it will contains nested block.
    Please reference the framework.proto for details.

    Notes: we have default_startup_program and default_main_program
    by default, a pair of them will shared the parameters.
    The default_startup_program only run once to initialize parameters,
Y
yuyang18 已提交
1527
    default_main_program run in every mini batch and adjust the weights.
D
dzhwinter 已提交
1528 1529

    Returns:
Y
yuyang18 已提交
1530
        A empty program.
D
dzhwinter 已提交
1531 1532

    Examples:
Y
yuyang18 已提交
1533 1534 1535 1536 1537 1538
        >>> main_program = fluid.Program()
        >>> startup_program = fluid.Program()
        >>> with fluid.program_guard(main_program=main_program, startup_program=startup_program):
        >>>     fluid.layers.data(name="x", shape=[-1, 784], dtype='float32')
        >>>     fluid.layers.data(name="y", shape=[-1, 1], dtype='int32')
        >>>     fluid.layers.fc(name="fc", shape=[10], dtype='float32', act="relu")
D
dzhwinter 已提交
1539 1540 1541

    """

1542 1543
    def __init__(self):
        self.desc = core.ProgramDesc()
Y
Yu Yang 已提交
1544 1545
        self.blocks = [Block(self, 0)]
        self.current_block_idx = 0
D
dzhwinter 已提交
1546
        self._seed = 0
Y
yuyang18 已提交
1547
        self._current_role = core.op_proto_and_checker_maker.OpRole.Forward
Y
yuyang18 已提交
1548
        self._op_role_var = []
T
tangwei12 已提交
1549 1550 1551 1552

        # for distribute
        self._is_distributed = False
        self._is_chief = False
T
tangwei12 已提交
1553
        self._slice_vars_and_attrs = []
T
tangwei12 已提交
1554
        self._endpoints = []
1555
        self._trainers_endpoints = []
T
tangwei12 已提交
1556
        self._distributed_lookup_table = None
Y
yuyang18 已提交
1557 1558 1559

    @property
    def op_role(self):
Y
yuyang18 已提交
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572
        """
        The operator role. In a enum {Forward, Backward, Optimize}.

        Notes: this is a low level API. It is used only for ParallelExecutor to
        duplicate or schedule operator to devices.

        For example, the forward operator should be executed on every device.
        The backward operator should be executed on every device and the
        parameter gradient of backward (use :code:`op_role_var` to get this
        variable) operator should be merged to one device. The optimization
        operators should be executed on only one device and broadcast the
        optimization result, i.e., the new parameter, to every other device.
        """
Y
yuyang18 已提交
1573 1574 1575 1576 1577 1578 1579 1580
        return self._current_role

    @op_role.setter
    def set_op_role(self, role):
        self._current_role = role

    @property
    def op_role_var(self):
Y
yuyang18 已提交
1581 1582 1583 1584 1585 1586 1587
        """
        The auxiliary variables for :code:`op_role` property.

        See Also: :code:`Program.op_role`'s documentation for details.

        Notes: This is a very low-level API. Users should not use it directly.
        """
Y
yuyang18 已提交
1588 1589 1590 1591
        return self._op_role_var

    @op_role_var.setter
    def set_op_role_var(self, var_name):
Y
yuyang18 已提交
1592
        self._op_role_var = [var_name]
Y
yuyang18 已提交
1593 1594

    @contextlib.contextmanager
W
Wu Yi 已提交
1595
    def _optimized_guard(self, param_and_grads):
Y
yuyang18 已提交
1596 1597 1598 1599 1600 1601 1602
        """
        A with guard to set :code:`Optimization` :code:`OpRole` and
        :code:`OpRoleVar` automatically.

        Notes: This is a very low level API. Users should not use it directly.

        Args:
1603
            param_and_grads(list): The variables (names) to be optimized.
Y
yuyang18 已提交
1604 1605 1606 1607

        Examples:

            >>> p, g = backward(...)
W
Wu Yi 已提交
1608
            >>> with program._optimized_guard([p,g]):
Y
yuyang18 已提交
1609 1610
            >>>     p = p - 0.001 * g
        """
X
Xin Pan 已提交
1611 1612 1613
        tmp_role = self._current_role
        tmp_var = self._op_role_var

Y
yuyang18 已提交
1614 1615
        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.Optimize
1616 1617 1618 1619
        self._op_role_var = [
            var.name if isinstance(var, Variable) else var
            for var in param_and_grads
        ]
Y
yuyang18 已提交
1620
        yield
X
Xin Pan 已提交
1621 1622
        self._op_role_var = tmp_var
        self._current_role = tmp_role
Y
Yu Yang 已提交
1623

1624
    @contextlib.contextmanager
X
Xin Pan 已提交
1625
    def _lr_schedule_guard(self, is_with_opt=False):
1626 1627 1628 1629 1630 1631 1632
        """
        A with guard to set :code:`LRSched` :code:`OpRole` and
        :code:`OpRoleVar` automatically. The :code:`OpRoleVar` is
        set to the target learning rate.

        Notes: This is a very low level API. Users should not use it directly.

X
Xin Pan 已提交
1633 1634 1635 1636
        Args:
            is_with_opt: Only set to true if these ops a in the middle
                 of a bunch of optimize ops so that it can be treated
                 correctly. For example, sgd->lr_op->sgd->lr_op->sgd.
1637 1638 1639 1640 1641 1642 1643

        Examples:

            >>> p, g = backward(...)
            >>> with program.lr_schedule_guard():
            >>>     lr = lr * decay
        """
1644 1645 1646 1647

        tmp_role = self._current_role
        tmp_var = self._op_role_var

1648 1649
        OpRole = core.op_proto_and_checker_maker.OpRole
        self._current_role = OpRole.LRSched
X
Xin Pan 已提交
1650 1651
        if is_with_opt:
            self._current_role = int(OpRole.LRSched) | int(OpRole.Optimize)
1652 1653 1654
        # TODO(typhoonzero): how to set target learning rate var
        self._op_role_var = []
        yield
1655 1656
        self._op_role_var = tmp_var
        self._current_role = tmp_role
1657

1658
    def __str__(self):
Y
yuyang18 已提交
1659 1660 1661 1662 1663 1664 1665 1666 1667
        """
        Get the protobuf debug string of this Program.

        Returns:
            (str): The protobuf debug string.

        Raises:
            ValueError: If any of required fields is not set.
        """
Y
Yang Yang(Tony) 已提交
1668 1669
        return self.to_string(True)

F
fengjiayi 已提交
1670 1671 1672
    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.
Y
yuyang18 已提交
1673

F
fengjiayi 已提交
1674
        Args:
Y
yuyang18 已提交
1675 1676
            throw_on_error(bool): raise Value error when any of required fields
                is not set.
F
fengjiayi 已提交
1677

Y
yuyang18 已提交
1678 1679 1680 1681
            with_details(bool): True if more details about variables and
                parameters, e.g., :code:`trainable`, :code:`optimize_attr`, need
                to print.

H
haowang101779990 已提交
1682 1683
        Returns:
            str : The debug string.
Y
yuyang18 已提交
1684 1685 1686 1687

        Raises:
            ValueError: If any of required fields is not set and throw_on_error is
                True.
F
fengjiayi 已提交
1688 1689 1690 1691 1692 1693 1694 1695 1696 1697

        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
            res_str = ""
            for block in self.blocks:
                res_str += block.to_string(throw_on_error, with_details)
        else:
            protostr = self.desc.serialize_to_string()
1698 1699
            proto = framework_pb2.ProgramDesc.FromString(
                six.binary_type(protostr))
F
fengjiayi 已提交
1700 1701
            res_str = _debug_string_(proto, throw_on_error)
        return res_str
1702

W
Wu Yi 已提交
1703
    def _get_desc(self):
Y
yuyang18 已提交
1704 1705 1706 1707 1708 1709 1710
        """
        Get the C++ side of `ProgramDesc` object pointer. The C++ object is
        exposed by :code:`pybind`.

        Notes: This is a very low level API. Users should not use this API
        directly.
        """
1711 1712
        return self.desc

X
version  
Xin Pan 已提交
1713 1714 1715
    def _version(self):
        return self.desc._version()

1716
    def clone(self, for_test=False):
Y
yuyang18 已提交
1717 1718 1719
        """
        Create a new, duplicated program.

1720

Y
yuyang18 已提交
1721 1722 1723 1724
        Some operators, e.g., :code:`batch_norm`, behave differently between
        training and testing. They have an attribute, :code:`is_test`, to
        control this behaviour. This method will change the :code:`is_test`
        attribute of them to :code:`True` when :code:`for_test=True`.
1725

Y
yuyang18 已提交
1726 1727 1728 1729
        * Set for_test to False when we want to clone the program for training.
        * Set for_test to True when we want to clone the program for testing.

        Notes: This API DOES NOT prune any operator. Use
L
Luo Tao 已提交
1730 1731 1732 1733 1734
        :code:`clone(for_test=True)` before backward and optimization please. e.g.

            >>> test_program = fluid.default_main_program().clone(for_test=True)
            >>> optimizer = fluid.optimizer.Momentum(learning_rate=0.01, momentum=0.9)
            >>> optimizer.minimize()
1735 1736

        Args:
Y
yuyang18 已提交
1737 1738
            for_test(bool): True if change the :code:`is_test` attribute of
                operators to :code:`True`.
1739

D
dzhwinter 已提交
1740
        Returns:
Y
yuyang18 已提交
1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
            Program: The new, duplicated Program object.

        Examples:

            1. To clone a test program, the sample code is:

            >>> import paddle.fluid as fluid
            >>> train_program = fluid.Program()
            >>> startup_program = fluid.Program()
            >>> with fluid.program_guard(train_program, startup_program):
            >>>     img = fluid.layers.data(name='image', shape=[784])
            >>>     hidden = fluid.layers.fc(input=img, size=200, act='relu')
            >>>     hidden = fluid.layers.dropout(hidden, dropout_prob=0.5)
            >>>     loss = fluid.layers.cross_entropy(
            >>>                 input=fluid.layers.fc(hidden, size=10, act='softmax'),
            >>>                 label=fluid.layers.data(name='label', shape=[1], dtype='int64'))
            >>>
            >>> test_program = train_program.clone(for_test=True)
            >>>
            >>> sgd = fluid.optimizer.SGD(learning_rate=1e-3)
            >>> with fluid.program_guard(train_program, startup_program):
            >>>     sgd.minimize(loss)

            2. The :code:`clone` method can be avoid if you create program for
            training and program for testing individually.

            >>> import paddle.fluid as fluid
            >>>
            >>> def network(is_test):
            >>>     img = fluid.layers.data(name='image', shape=[784])
            >>>     hidden = fluid.layers.fc(input=img, size=200, act='relu')
            >>>     hidden = fluid.layers.dropout(hidden, dropout_prob=0.5, is_test=is_test)
            >>>     loss = fluid.layers.cross_entropy(
            >>>                 input=fluid.layers.fc(hidden, size=10, act='softmax'),
            >>>                 label=fluid.layers.data(name='label', shape=[1], dtype='int64'))
            >>>     return loss
            >>>
            >>> train_program = fluid.Program()
            >>> startup_program = fluid.Program()
            >>> test_program = fluid.Program()
            >>>
            >>> with fluid.program_guard(train_program, startup_program):
            >>>     with fluid.unique_name.guard():
            >>>         loss = network(is_test=False)
            >>>         sgd = fluid.optimizer.SGD(learning_rate=1e-3)
            >>>         sgd.minimize(loss)
            >>>
            >>> # the test startup program is not used.
            >>> with fluid.program_guard(test_program, fluid.Program()):
            >>>     with fluid.unique_name.guard():
            >>>         loss = network(is_test=True)

            The two code snippets above will generate same programs.
1794 1795
        """
        if for_test:
X
Xin Pan 已提交
1796
            p = self._inference_optimize(prune_read_op=False)
1797
        else:
1798
            p = Program()
G
gongweibao 已提交
1799 1800
            p.current_block_idx = self.current_block_idx
            p._seed = self._seed
1801
            p.desc = core.ProgramDesc(self.desc)
M
minqiyang 已提交
1802 1803 1804
            p.blocks = [
                Block(p, i) for i in six.moves.range(self.desc.num_blocks())
            ]
G
gongweibao 已提交
1805 1806 1807 1808

            p._current_role = self._current_role
            p._op_role_var = self._op_role_var

W
Wu Yi 已提交
1809
            p._sync_with_cpp()
1810

W
Wu Yi 已提交
1811
        p._copy_param_info_from(self)
W
Wu Yi 已提交
1812
        p._copy_data_info_from(self)
1813
        p._copy_dist_param_info_from(self)
Y
Yu Yang 已提交
1814
        return p
1815

W
Wu Yi 已提交
1816
    def _prune(self, targets):
Y
yuyang18 已提交
1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831
        """
        Prune operators and variables which are not needed to generate
        :code:`targets`.

        Notes: This is a very low level API. Users should not use this API
        directly. This API is in flux and not stable.

        Args:
            targets(list|Variable|Operator): A list of variables or operators
                need to be pruned

        Returns:
            Program:  A new, pruned program.

        """
1832 1833 1834 1835 1836 1837
        if not isinstance(targets, list):
            targets = [targets]
        targets_idx = []
        for t in targets:
            if not isinstance(t, Operator):
                if isinstance(t, Variable):
1838 1839
                    # After transpiler processing, the op that output this
                    # variable maybe has been changed, so t.op is not reliable
1840
                    # and we need to find the current op that generate this
1841 1842 1843 1844 1845 1846 1847 1848
                    # variable here.
                    t.op = None
                    global_block = self.global_block()
                    for idx, op in enumerate(global_block.ops):
                        if t.name in op.output_arg_names:
                            t.op = op
                            break

1849
                    t = t.op
1850 1851 1852 1853
                    if t is None:
                        raise ValueError(
                            "The target variable must have an "
                            "associated operator that generates it.")
1854
                else:
1855 1856
                    raise ValueError("All targets of prune() can only be "
                                     "Variable or Operator.")
1857 1858 1859 1860

            targets_idx.append([t.block.idx, t.idx])
        res = Program()
        res.desc = core.prune(self.desc, targets_idx)
M
minqiyang 已提交
1861 1862 1863
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
W
Wu Yi 已提交
1864
        res._sync_with_cpp()
1865 1866
        return res

X
Xin Pan 已提交
1867
    def _inference_optimize(self, prune_read_op=True):
Y
yuyang18 已提交
1868
        """
F
fengjiayi 已提交
1869 1870 1871 1872 1873
        This method will create a new program and do following adjustments on it:
        1. Remove all reader variables and their creator ops if exist.

        2. Remove the :code:`read_op` if exists.

1874
        3. change the :code:`is_test`
Y
yuyang18 已提交
1875 1876 1877
        attribute of operators to :code:`True`. All the :code:`Parameter`
        information will be lost.

1878
        Args:
X
Xin Pan 已提交
1879 1880
            prune_read_op(bool): remove the read ops that are added by py_reader
                                 for cpp inference library
1881

Y
yuyang18 已提交
1882 1883 1884 1885 1886 1887
        Notes: This API is a very low level API. Use
        :code:`Program.clone(for_test=True)` instead.

        Returns:
            Program: The new program.
        """
1888
        res = Program()
1889
        res.desc = core.ProgramDesc(self.desc)
F
fengjiayi 已提交
1890 1891 1892 1893

        # remove all readers and the read_op if exist
        read_op_idx = 0
        root_block = res.desc.block(0)
X
Xin Pan 已提交
1894
        if prune_read_op:
1895 1896 1897 1898 1899 1900 1901 1902 1903
            while True:
                if read_op_idx >= root_block.op_size() or root_block.op(
                        read_op_idx).type() == 'read':
                    break
                read_op_idx += 1
            if read_op_idx < root_block.op_size():
                root_block._remove_op(0, read_op_idx + 1)
            for var in root_block.all_vars():
                if var.type() == core.VarDesc.VarType.READER:
M
minqiyang 已提交
1904
                    root_block._remove_var(cpt.to_bytes(var.name()))
F
fengjiayi 已提交
1905 1906

        # change all `is_test` attributes to True
M
minqiyang 已提交
1907
        for i in six.moves.range(res.desc.num_blocks()):
1908
            block = res.desc.block(i)
M
minqiyang 已提交
1909
            for j in six.moves.range(block.op_size()):
1910 1911
                op = block.op(j)
                if op.has_attr('is_test'):
W
Wu Yi 已提交
1912
                    op._set_attr('is_test', True)
M
minqiyang 已提交
1913 1914 1915
        res.blocks = [
            Block(res, i) for i in six.moves.range(res.desc.num_blocks())
        ]
W
Wu Yi 已提交
1916
        res._sync_with_cpp()
1917 1918
        return res

1919 1920
    @staticmethod
    def parse_from_string(binary_str):
Y
yuyang18 已提交
1921 1922 1923 1924 1925 1926 1927
        """
        Deserialize a program desc from protobuf binary string.

        Notes: All information about parameters will be lost after serialization
        and deserialization.

        Args:
1928
            binary_str_type(str): The binary prootbuf string.
Y
yuyang18 已提交
1929 1930 1931 1932

        Returns:
            Program: A deserialized program desc.
        """
1933 1934
        p = Program()
        p.desc = core.ProgramDesc(binary_str)
M
minqiyang 已提交
1935
        p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())]
W
Wu Yi 已提交
1936
        p._sync_with_cpp()
1937
        return p
Y
Yu Yang 已提交
1938

D
dzhwinter 已提交
1939 1940
    @property
    def random_seed(self):
Y
yuyang18 已提交
1941 1942 1943 1944 1945 1946
        """
        The default random seed for random operators in Program. Zero means get
        the random seed from random device.

        Notes: It must be set before the operators have been added.
        """
D
dzhwinter 已提交
1947 1948
        return self._seed

Q
qiaolongfei 已提交
1949 1950
    @property
    def num_blocks(self):
Y
yuyang18 已提交
1951 1952 1953
        """
        The number of blocks in this program.
        """
Q
qiaolongfei 已提交
1954 1955
        return self.desc.num_blocks()

D
dzhwinter 已提交
1956 1957 1958 1959 1960 1961
    @random_seed.setter
    def random_seed(self, seed):
        if not isinstance(seed, int):
            raise ValueError("Seed must be a integer.")
        self._seed = seed

Y
Yu Yang 已提交
1962
    def __repr__(self):
1963
        return self.__str__()
1964

Y
Yu Yang 已提交
1965
    def global_block(self):
Y
yuyang18 已提交
1966 1967 1968
        """
        Get the first block of this program.
        """
Y
Yu Yang 已提交
1969 1970
        return self.blocks[0]

Q
Qiao Longfei 已提交
1971
    def block(self, index):
Y
yuyang18 已提交
1972 1973 1974 1975 1976 1977 1978 1979
        """
        Get the :code:`index` block of this program
        Args:
            index(int): The index of block to get

        Returns:
            Block: The :code:`index` block
        """
Q
Qiao Longfei 已提交
1980 1981
        return self.blocks[index]

Y
Yu Yang 已提交
1982
    def current_block(self):
Y
yuyang18 已提交
1983 1984 1985 1986
        """
        Get the current block. The :code:`current` block is the block to append
        operators.
        """
Y
Yu Yang 已提交
1987 1988
        return self.blocks[self.current_block_idx]

W
Wu Yi 已提交
1989
    def _create_block(self, parent_idx=None):
Y
yuyang18 已提交
1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
        """
        Create a new block with the :code:`parent_idx` and change the current block
        to new block.

        Args:
            parent_idx(int): The parent block index.

        Returns:
            Block: The new block.
        """
Y
Yu Yang 已提交
2000
        new_block_idx = len(self.blocks)
F
update  
fengjiayi 已提交
2001 2002 2003
        parent = self.current_block() if parent_idx is None else self.block(
            parent_idx)
        self.desc.append_block(parent.desc)
Y
Yu Yang 已提交
2004 2005 2006 2007
        self.current_block_idx = new_block_idx
        self.blocks.append(Block(self, self.current_block_idx))
        return self.current_block()

W
Wu Yi 已提交
2008
    def _rollback(self):
Y
yuyang18 已提交
2009 2010 2011 2012 2013
        """
        Exit a code block, i.e., roll back to the parent block.
        Returns:
            None
        """
Y
Yu Yang 已提交
2014 2015
        self.current_block_idx = self.current_block().parent_idx

W
Wu Yi 已提交
2016
    def _sync_with_cpp(self):
Y
yuyang18 已提交
2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
        """
        Synchronize Python instance to its binding C++ object instance.
        If the program is modified in C++ space, this method should be invoked.

        Notes: This is a very low level API. Users should not invoke it
        directly.

        Returns:
            None
        """
Q
Qiao Longfei 已提交
2027 2028 2029
        for block_idx in range(len(self.blocks), self.desc.num_blocks()):
            self.blocks.append(Block(self, block_idx))
        for block in self.blocks:
W
Wu Yi 已提交
2030
            block._sync_with_cpp()
Q
Qiao Longfei 已提交
2031

W
Wu Yi 已提交
2032
    def _copy_param_info_from(self, other):
2033
        """
2034
        Copy the information of parameters from other program.
D
dzhwinter 已提交
2035

Y
yuyang18 已提交
2036 2037 2038
        Notes: This is a very low level API. Users should not invoke it
        directly.

2039 2040 2041 2042 2043 2044 2045
        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
W
Wu Yi 已提交
2046
            raise TypeError("_copy_param_info_from should be invoked with "
2047 2048 2049
                            "Program")

        if len(self.blocks) != len(other.blocks):
W
Wu Yi 已提交
2050
            raise ValueError("_copy_param_info_from should be invoked with two "
2051
                             "program, with represent the same topology")
W
Wu Yi 已提交
2052
        self.global_block()._copy_param_info_from(other.global_block())
2053

2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072
    def _copy_dist_param_info_from(self, other):
        """
        Copy the information of distributed information from other program.

        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
            raise TypeError("_copy_dist_param_info_from should be invoked with "
                            "Program")
        self._is_distributed = other._is_distributed
        self._is_chief = other._is_chief
        self._slice_vars_and_attrs = other._slice_vars_and_attrs
        self._endpoints = other._endpoints
        self._distributed_lookup_table = other._distributed_lookup_table

W
Wu Yi 已提交
2073
    def _copy_data_info_from(self, other):
F
fengjiayi 已提交
2074 2075
        """
        Copy the information of data variables from other program.
D
dzhwinter 已提交
2076

Y
yuyang18 已提交
2077 2078 2079
        Notes: This is a very low level API. Users should not invoke it
        directly.

F
fengjiayi 已提交
2080 2081 2082 2083 2084 2085 2086
        Args:
            other(Program): Other program

        Returns:
            None
        """
        if not isinstance(other, Program):
W
Wu Yi 已提交
2087
            raise TypeError("_copy_param_info_from should be invoked with "
F
fengjiayi 已提交
2088 2089 2090
                            "Program")

        if len(self.blocks) != len(other.blocks):
W
Wu Yi 已提交
2091
            raise ValueError("_copy_param_info_from should be invoked with two "
F
fengjiayi 已提交
2092
                             "program, with represent the same topology")
2093
        for var in list(other.global_block().vars.values()):
F
fengjiayi 已提交
2094 2095 2096
            if var.is_data:
                self.global_block().var(var.name).is_data = True

2097
    def list_vars(self):
Y
yuyang18 已提交
2098 2099 2100 2101 2102 2103
        """
        Get all variables from this Program. A iterable object is returned.

        Returns:
            iterable: The generator will yield every variable in this program.
        """
2104
        for each_block in self.blocks:
2105
            for each_var in list(each_block.vars.values()):
2106 2107
                yield each_var

Y
Yu Yang 已提交
2108

Y
Yu Yang 已提交
2109
class Parameter(Variable):
2110
    """
2111
    Parameter is derived from Variable. A parameter is a persistable
2112
    Variable, and will be updated by optimizers after each iteration.
2113
    The training of a neural network is essentially the updating of
2114 2115
    its parameters.

2116
    Relative to a general Variable, a Parameter has several its own
2117 2118
    member variables:

2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
    Args:
        trainable(bool): True if the parameter need to be updated after
            iterations.
        optimize_attr(map): Parameter attributes related with optimizing.
            Currently, it only contains 'learning_rate'.
            Default: {'learning_rate': 1.0}
        regularizer(WeightDecayRegularizer): The Regularizer which will
            be applied on the parameter. Default: None
        gradient_clip_attr(BaseGradientClipAttr): The gradint clip strategy
            which will be applied on the parameter. Default: None
        do_model_average(bool): True if the model average strategy will
            be applied on this parameter.
2131 2132
    """

Y
Yu Yang 已提交
2133 2134 2135 2136 2137 2138 2139 2140 2141 2142
    def __init__(self, block, shape, dtype, **kwargs):
        if shape is None or dtype is None:
            raise ValueError("Parameter must set shape and dtype")
        if len(shape) == 0:
            raise ValueError("Parameter shape cannot be empty")

        for each in shape:
            if each < 0:
                raise ValueError("Parameter shape should not be related with "
                                 "batch-size")
2143 2144 2145

        Variable.__init__(
            self, block, persistable=True, shape=shape, dtype=dtype, **kwargs)
Y
Yu Yang 已提交
2146 2147 2148 2149
        self.trainable = kwargs.get('trainable', True)

        self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})

2150 2151
        self.regularizer = kwargs.get('regularizer', None)

F
fengjiayi 已提交
2152
        self.gradient_clip_attr = kwargs.get('gradient_clip_attr', None)
Y
Yu Yang 已提交
2153

W
wanghaoshuang 已提交
2154
        self.do_model_average = kwargs.get('do_model_average', None)
W
wanghaoshuang 已提交
2155

F
fengjiayi 已提交
2156 2157 2158
    def __str__(self):
        return self.to_string(True)

F
update  
fengjiayi 已提交
2159 2160 2161
    def to_string(self, throw_on_error, with_details=False):
        """
        To debug string.
D
dzhwinter 已提交
2162

F
update  
fengjiayi 已提交
2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176
        Args:
            throw_on_error(bool): raise exception when self is not initialized
                when throw_on_error is True
            with_details(bool): more details about variables and parameters
                (e.g. trainable, optimize_attr, ...) will be printed when with_details is True

        Returns(str): The debug string.

        """
        assert isinstance(throw_on_error, bool) and isinstance(with_details,
                                                               bool)
        if with_details:
            res_str = Variable.to_string(self, throw_on_error, True)
            additional_attr = ("trainable", "optimize_attr", "regularizer",
W
wanghaoshuang 已提交
2177
                               "gradient_clip_attr", "do_model_average")
F
update  
fengjiayi 已提交
2178
            for attr_name in additional_attr:
2179 2180
                res_str += "%s: %s\n" % (
                    attr_name, six.binary_type(getattr(self, attr_name)))
F
update  
fengjiayi 已提交
2181 2182
        else:
            res_str = Variable.to_string(self, throw_on_error, False)
F
fengjiayi 已提交
2183 2184 2185 2186
        return res_str

    __repr__ = __str__

Y
Yu Yang 已提交
2187

Y
Yu Yang 已提交
2188
# program is a global instance.
Y
Yu Yang 已提交
2189 2190
_main_program_ = Program()
_startup_program_ = Program()
2191

2192

2193
def default_startup_program():
Y
Yu Yang 已提交
2194
    """
Y
yuyang18 已提交
2195 2196 2197 2198 2199 2200 2201 2202 2203
    Get default/global startup program.

    The layer function in :code:`fluid.layers` will create parameters, readers,
    NCCL handles as global variables. The :code:`startup_program` will
    initialize them by the operators in startup program. The layer function will
    append these initialization operators into startup program.

    This method will return the :code:`default` or the :code:`current` startup
    program. Users can use :code:`fluid.program_guard` to switch program.
2204

Y
Yu Yang 已提交
2205 2206 2207
    Returns:
        Program: startup program
    """
Y
Yu Yang 已提交
2208
    return _startup_program_
2209

2210

2211
def default_main_program():
Y
Yu Yang 已提交
2212
    """
Y
yuyang18 已提交
2213 2214 2215 2216 2217 2218 2219 2220 2221
    Get default/global main program. The main program is used for training or
    testing.

    All layer function in :code:`fluid.layers` will append operators and
    variables to the :code:`default_main_program`.

    The :code:`default_main_program` is the default program in a lot of APIs.
    For example, the :code:`Executor.run()` will execute the
    :code:`default_main_program` when the program is not specified.
2222

Y
Yu Yang 已提交
2223 2224 2225
    Returns:
        Program: main program
    """
Y
Yu Yang 已提交
2226
    return _main_program_
Y
Yu Yang 已提交
2227 2228 2229 2230 2231


def switch_main_program(program):
    """
    Switch the main program to a new program.
2232

Y
Yu Yang 已提交
2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246
    Args:
        program(Program): The new main program

    Returns:
        Program: The previous main program
    """
    global _main_program_
    prev_program = _main_program_
    _main_program_ = program
    return prev_program


def switch_startup_program(program):
    """
2247
    Switch the startup program to a new program
Y
Yu Yang 已提交
2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262
    Args:
        program(Program): The new startup program

    Returns:
        Program: The previous startup program
    """
    global _startup_program_
    prev_program = _startup_program_
    _startup_program_ = program
    return prev_program


@contextlib.contextmanager
def program_guard(main_program, startup_program=None):
    """
Y
yuyang18 已提交
2263 2264 2265
    Change the global main program and startup program with `with` statement.
    Layer functions in the Python `with` block will append operators and
    variables to the new main programs.
2266

Y
Yu Yang 已提交
2267
    Examples:
Y
yuyang18 已提交
2268 2269 2270 2271 2272 2273 2274 2275 2276 2277

        >>> import paddle.fluid as fluid
        >>> main_program = fluid.Program()
        >>> startup_program = fluid.Program()
        >>> with fluid.program_guard(main_program, startup_program):
        >>>     data = fluid.layers.data(...)
        >>>     hidden = fluid.layers.fc(...)

    Notes: The temporary :code:`Program` can be used if the user does not need
    to construct either of startup program or main program.
2278

Y
Yu Yang 已提交
2279
    Examples:
Y
yuyang18 已提交
2280 2281 2282 2283 2284 2285

        >>> import paddle.fluid as fluid
        >>> main_program = fluid.Program()
        >>> # does not care about startup program. Just pass a temporary value.
        >>> with fluid.program_guard(main_program, fluid.Program()):
        >>>     data = ...
2286

Y
Yu Yang 已提交
2287
    Args:
Y
yuyang18 已提交
2288
        main_program(Program): New main program inside `with` statement.
2289
        startup_program(Program): New startup program inside `with` statement.
Y
Yu Yang 已提交
2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302
            None means do not change startup program.
    """
    if not isinstance(main_program, Program):
        raise TypeError("main_program should be Program")
    main_program = switch_main_program(main_program)
    if startup_program is not None:
        if not isinstance(startup_program, Program):
            raise TypeError("startup_program should be Program")
        startup_program = switch_startup_program(startup_program)
    yield
    switch_main_program(main_program)
    if startup_program is not None:
        switch_startup_program(startup_program)
X
xuwei06 已提交
2303 2304


W
Wu Yi 已提交
2305
def _get_var(name, program=None):
X
xuwei06 已提交
2306
    """
Y
yuyang18 已提交
2307
    Get a variable by name from the global block of a program.
F
fengjiayi 已提交
2308

X
xuwei06 已提交
2309 2310 2311
    Args:
        name(str): name of the variable
        program(Program|None): program object.
T
tangwei12 已提交
2312
        If None, default_global_program() will be used.
X
xuwei06 已提交
2313 2314 2315 2316 2317 2318 2319

    Returns:
        Variable
    """
    if program is None:
        program = default_main_program()
    assert isinstance(name, str)
2320
    assert isinstance(program, Program)
X
xuwei06 已提交
2321 2322

    return program.global_block().var(name)
2323 2324 2325 2326 2327 2328 2329 2330 2331


@contextlib.contextmanager
def _imperative_guard(tracer):
    global _imperative_tracer_
    tmp_trace = _imperative_tracer_
    _imperative_tracer_ = tracer
    yield
    _imperative_tracer_ = tmp_trace