control_flow.py 145.6 KB
Newer Older
1
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
14 15

from __future__ import print_function
S
rename  
sneaxiy 已提交
16
from ..wrapped_decorator import signature_safe_contextmanager
D
dzhwinter 已提交
17

18
from .layer_function_generator import autodoc, templatedoc
19
from .tensor import assign, cast, fill_constant
20
from .. import core
21
from ..framework import Program, Variable, Operator, in_dygraph_mode
22
from ..layer_helper import LayerHelper, unique_name
M
minqiyang 已提交
23
from .nn import logical_and, logical_not, logical_or
24
from .utils import assert_same_structure, map_structure, hold_mutable_vars, copy_mutable_vars
Y
yuyang18 已提交
25
import numpy
26
import warnings
27
import six
L
liym27 已提交
28
from functools import reduce, partial
29
from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type
30 31
from ... import compat as cpt
from ..backward import _infer_var_data_type_shape_
D
dzhwinter 已提交
32

Q
QI JUN 已提交
33
__all__ = [
W
Wu Yi 已提交
34
    'While', 'Switch', 'increment', 'array_write', 'create_array', 'less_than',
Z
zhoukunsheng 已提交
35
    'less_equal', 'greater_than', 'greater_equal', 'equal', 'not_equal',
36
    'array_read', 'array_length', 'cond', 'IfElse', 'DynamicRNN', 'StaticRNN',
G
guofei 已提交
37 38
    'reorder_lod_tensor_by_rank', 'Print', 'is_empty', 'case', 'switch_case',
    'while_loop'
D
dzhwinter 已提交
39 40
]

Y
Yu Yang 已提交
41

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
def select_output(input, outputs, mask):
    """
    **select_output**    
    This API takes in one input and multiple outputs and an integer mask. It
    selects the output specified by the mask and copy the input to selected
    output. It is useful in control flow.

    Args:
        input(Variable): The input variable
        outputs(tuple|list): The output variables
        mask(Variable): A tensor containing 1 integer number selecting which
            output to be copied with input

    Returns:
        Variable: The outputs variables
    """
    helper = LayerHelper('select_output', **locals())
59 60 61 62
    check_type(input, 'input', (Variable), 'select_output')
    check_variable_and_dtype(mask, 'mask', ['int32'], 'select_output')
    check_type(outputs, 'outputs', (list, tuple), 'select_output')

63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
    helper.append_op(
        type='select_output',
        inputs={'X': input,
                'Mask': mask},
        outputs={'Out': outputs})
    return outputs


def select_input(inputs, mask):
    """
    **select_input**
    
    This API takes in multiple inputs and uses an integer mask to select one
    input to output. It is useful in control flow.

    Args:
        inputs(tuple|list): The input variables
        mask(Variable): A tensor containing 1 integer number selecting which
            input to output

    Returns:
        Variable: The selected input variable
    """
    helper = LayerHelper('select_input', **locals())
87 88 89 90 91 92
    check_type(inputs, 'inputs', (list, tuple), 'select_input')
    check_variable_and_dtype(mask, 'mask', ['int32'], 'select_input')

    input_dtype = inputs[0].dtype
    input_shape = inputs[0].shape
    input_type = inputs[0].type
93 94 95

    out = helper.create_variable(
        dtype=input_dtype, shape=input_shape, type=input_type)
96 97 98 99 100 101 102 103
    helper.append_op(
        type='select_input',
        inputs={'X': inputs,
                'Mask': mask},
        outputs={'Out': out})
    return out


104
def split_lod_tensor(input, mask, level=0):
105 106 107 108
    """
    This function takes in an input that contains the complete lod information,
    and takes in a mask which is used to mask certain parts of the input.
    The output is the true branch and the false branch with the mask applied to
Q
qiaolongfei 已提交
109 110
    the input at a certain level in the tensor. Mainly used in IfElse to split
    data into two parts.
111 112

    Args:
113
        input(Variable|tuple|list|None): The input tensor that contains complete
114
                                lod information needed to construct the output.
115
        mask(Variable|list): A bool column vector which masks the input.
Q
qiaolongfei 已提交
116
        level(int): The specific lod level to split.
117 118

    Returns:
Q
qiaolongfei 已提交
119 120 121 122
        tuple(Variable, Variable):
        The true branch of tensor as per the mask applied to input.

        The false branch of tensor as per the mask applied to input.
123 124 125 126

    Examples:
        .. code-block:: python

127
          import paddle.fluid as fluid
Q
qiaolongfei 已提交
128
          x = fluid.layers.data(name='x', shape=[1])
129 130
          x.persistable = True

Q
qiaolongfei 已提交
131
          y = fluid.layers.data(name='y', shape=[1])
132 133
          y.persistable = True

Q
qiaolongfei 已提交
134
          out_true, out_false = fluid.layers.split_lod_tensor(
135
                input=x, mask=y, level=level)
136

137
    """
138 139 140 141
    check_type(input, 'input', (Variable, list, tuple, type(None)),
               'fluid.layers.split_lod_tensor')
    check_type(mask, 'mask', (Variable, list), 'fluid.layers.split_lod_tensor')
    check_type(level, 'level', int, 'fluid.layers.split_lod_tensor')
142
    helper = LayerHelper('split_lod_tensor', **locals())
X
Xin Pan 已提交
143 144
    out_true = helper.create_variable_for_type_inference(dtype=input.dtype)
    out_false = helper.create_variable_for_type_inference(dtype=input.dtype)
145 146 147 148 149 150 151 152 153 154 155 156
    helper.append_op(
        type='split_lod_tensor',
        inputs={
            'X': input,
            'Mask': mask,
        },
        outputs={'OutTrue': out_true,
                 'OutFalse': out_false},
        attrs={'level': level})
    return out_true, out_false


157
def merge_lod_tensor(in_true, in_false, x, mask, level=0):
158 159 160 161 162
    """
    **merge_lod_tensor**

    This function takes in an input :math:`x`, the True branch, the False
    branch and a binary :math:`mask`. Using this information, this function
Q
qiaolongfei 已提交
163 164 165
    merges the True and False branches of the tensor into a single tensor as
    output at a certain lod level indicated by :math:`level`. Used in IfElse
    to merge the output if True block and False Block.
166 167 168 169 170 171 172

    Args:
        in_true(tuple|list|None): The True branch to be merged.
        in_false(tuple|list|None): The False branch to be merged.
        x(tuple|list|None): The input tensor that contains complete
                            lod information needed to construct the output.
        mask(list): A bool column vector which masks the input.
Q
qiaolongfei 已提交
173
        level(int): The specific lod level to merge.
174 175 176 177 178 179 180

    Returns:
        Variable: The merged output tensor.

    Examples:
        .. code-block:: python

181
          import paddle.fluid as fluid
182 183 184 185 186 187 188 189 190 191 192 193
          x = layers.data(
                      name='x', shape=[1], dtype='float32', stop_gradient=False)
          y = layers.data(
                name='y', shape=[1], dtype='bool', stop_gradient=False)

          level = 0

          out_true, out_false = layers.split_lod_tensor(
                input=x, mask=y, level=level)
          out = layers.merge_lod_tensor(
                in_true=out_true, in_false=out_false, mask=y, x=x, level=level)
    """
194
    helper = LayerHelper('merge_lod_tensor', **locals())
X
Xin Pan 已提交
195
    out = helper.create_variable_for_type_inference(dtype=in_true.dtype)
196 197 198 199 200 201 202 203 204 205 206
    helper.append_op(
        type='merge_lod_tensor',
        inputs={'X': x,
                'Mask': mask,
                'InTrue': in_true,
                'InFalse': in_false},
        outputs={'Out': out},
        attrs={'level': level})
    return out


Y
Yan Chunwei 已提交
207 208 209
def Print(input,
          first_n=-1,
          message=None,
210
          summarize=20,
Y
Yan Chunwei 已提交
211 212 213
          print_tensor_name=True,
          print_tensor_type=True,
          print_tensor_shape=True,
Y
yangyaming 已提交
214 215
          print_tensor_lod=True,
          print_phase='both'):
Y
Yan Chunwei 已提交
216 217 218 219 220 221 222 223 224 225
    '''
    **Print operator**

    This creates a print op that will print when a tensor is accessed.

    Wraps the tensor passed in so that whenever that a tensor is accessed,
    the message `message` is printed, along with the current value of the
    tensor `t`.

    Args:
Y
yangyaming 已提交
226
        input (Variable): A Tensor to print.
227
        summarize (int): Number of elements in the tensor to be print. If it's
T
tianshuo78520a 已提交
228
                value is -1, then all elements in the tensor will be print.
Y
yangyaming 已提交
229 230
        message (str): A string message to print as a prefix.
        first_n (int): Only log `first_n` number of times.
231 232 233 234
        print_tensor_name (bool, optional): Print the tensor name. Default: True.
        print_tensor_type (bool, optional): Print the tensor type. Defaultt: True.
        print_tensor_shape (bool, optional): Print the tensor shape. Default: True.
        print_tensor_lod (bool, optional): Print the tensor lod. Default: True.
235
        print_phase (str): Which phase to displace, including 'forward',
236 237 238
                'backward' and 'both'. Default: 'both'. If set to 'backward', will 
                only print the gradients of input tensor; If set to 'both', will
                both print the input tensor itself and the gradients of input tensor.
Y
Yan Chunwei 已提交
239 240

    Returns:
241
        Variable: Output tensor.
Y
Yan Chunwei 已提交
242

243 244 245 246
    NOTES:
        The input and output are two different variables, and in the
        following process, you should use the output variable but not the input,
        otherwise, the print layer doesn't have backward.
Y
Yan Chunwei 已提交
247

Y
Yan Chunwei 已提交
248 249
    Examples:
        .. code-block:: python
250 251 252
           
           import paddle.fluid as fluid
           
253 254 255 256 257 258
           input = fluid.layers.fill_constant(shape=[10,2], value=3, dtype='int64')
           input = fluid.layers.Print(input, message="The content of input layer:")
           
           main_program = fluid.default_main_program()
           exe = fluid.Executor(fluid.CPUPlace())
           exe.run(main_program)
Y
Yan Chunwei 已提交
259

260 261 262
    Output at runtime:
        .. code-block:: bash 
           
263
           The content of input layer:     The place is:CPUPlace
264 265 266 267 268
           Tensor[fill_constant_0.tmp_0]
               shape: [10,2,]
               dtype: x
               data: 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, 
               
Y
Yan Chunwei 已提交
269
    '''
270 271 272
    check_variable_and_dtype(input, 'input',
                             ['float32', 'float64', 'int32', 'int64', 'bool'],
                             'fluid.layers.Print')
273

274 275
    helper = LayerHelper('print' + "_" + input.name, **locals())
    output = helper.create_variable_for_type_inference(input.dtype)
Y
Yan Chunwei 已提交
276 277
    helper.append_op(
        type='print',
Y
yangyaming 已提交
278
        inputs={'In': input},
279
        outputs={'Out': output},
Y
Yan Chunwei 已提交
280 281 282 283 284 285 286 287
        attrs={
            'first_n': first_n,
            'summarize': summarize,
            'message': message or "",
            'print_tensor_name': print_tensor_name,
            'print_tensor_type': print_tensor_type,
            'print_tensor_shape': print_tensor_shape,
            'print_tensor_lod': print_tensor_lod,
Y
yangyaming 已提交
288
            'print_phase': print_phase.upper()
Y
Yu Yang 已提交
289
        })
290
    return output
Y
Yan Chunwei 已提交
291 292


Y
Yu Yang 已提交
293 294
class BlockGuard(object):
    """
295 296 297 298
    BlockGuard class.

    BlockGuard class is used to create a sub-block in a program by
    using the Python `with` keyword.
Y
Yu Yang 已提交
299 300
    """

301 302
    def __init__(self, main_program):
        if not isinstance(main_program, Program):
Y
Yu Yang 已提交
303
            raise TypeError("BlockGuard takes a program")
304
        self.main_program = main_program
Y
Yu Yang 已提交
305 306

    def __enter__(self):
W
Wu Yi 已提交
307
        self.main_program._create_block()
Y
Yu Yang 已提交
308 309

    def __exit__(self, exc_type, exc_val, exc_tb):
W
Wu Yi 已提交
310
        self.main_program._rollback()
Y
Yu Yang 已提交
311 312 313 314 315
        if exc_type is not None:
            return False  # re-raise exception
        return True


Y
Yang Yang 已提交
316 317 318 319 320
class BlockGuardWithCompletion(BlockGuard):
    """
    BlockGuardWithCompletion class.

    BlockGuardWithCompletion class is used to create an op with a block in a program.
321 322
    """

Y
Yu Yang 已提交
323
    def __init__(self, rnn):
X
Xin Pan 已提交
324
        if not isinstance(rnn, StaticRNN):
X
Xin Pan 已提交
325
            raise TypeError("BlockGuardWithCompletion takes a StaticRNN")
Y
Yang Yang 已提交
326
        super(BlockGuardWithCompletion, self).__init__(rnn.helper.main_program)
Y
Yu Yang 已提交
327 328 329 330
        self.rnn = rnn

    def __enter__(self):
        self.rnn.status = StaticRNN.IN_RNN_BLOCK
Y
Yang Yang 已提交
331
        return super(BlockGuardWithCompletion, self).__enter__()
Y
Yu Yang 已提交
332 333

    def __exit__(self, exc_type, exc_val, exc_tb):
Y
Yu Yang 已提交
334 335
        if exc_type is not None:
            return False
Y
Yu Yang 已提交
336
        self.rnn.status = StaticRNN.AFTER_RNN_BLOCK
337
        self.rnn._complete_op()
Y
Yang Yang 已提交
338 339
        return super(BlockGuardWithCompletion, self).__exit__(exc_type, exc_val,
                                                              exc_tb)
Y
Yu Yang 已提交
340 341 342 343


class StaticRNNMemoryLink(object):
    """
344 345 346 347
    StaticRNNMemoryLink class.

    StaticRNNMemoryLink class is used to create a link between two
    memory cells of a StaticRNN.
Y
yuyang18 已提交
348 349 350 351 352 353 354 355 356


    NOTE: This is a internal data structure of a very low-level API.
    Please use StaticRNN instead.

    Args:
        init(Variable): the initial variable for Memory.
        pre_mem(Variable): the memory variable in previous time step.
        mem(Variable): the memory variable in current time step.
Y
Yu Yang 已提交
357 358 359 360 361 362 363 364 365
    """

    def __init__(self, init, pre_mem, mem=None):
        self.init = init
        self.pre_mem = pre_mem
        self.mem = mem


class StaticRNN(object):
366 367 368
    """
    StaticRNN class.

369 370 371 372 373 374 375
    The StaticRNN can process a batch of sequence data. The first dimension of inputs
    represents sequence length, the length of each input sequence must be equal.
    StaticRNN will unfold sequence into time steps, user needs to define how to process
    each time step during the :code:`with` step.

    Args:
        name (str, optional): Please refer to :ref:`api_guide_Name`, Default None.
C
chengduo 已提交
376 377

    Examples:
378 379 380 381 382 383
        .. code-block:: python

            import paddle.fluid as fluid
            import paddle.fluid.layers as layers

            vocab_size, hidden_size=10000, 200
384 385
            x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64')
            # create word sequence
386 387 388 389 390
            x_emb = layers.embedding(
                input=x,
                size=[vocab_size, hidden_size],
                dtype='float32',
                is_sparse=False)
391
            # transform batch size to dim 1
392 393 394 395
            x_emb = layers.transpose(x_emb, perm=[1, 0, 2])

            rnn = fluid.layers.StaticRNN()
            with rnn.step():
396
                # mark created x_emb as input, each step process a word
397
                word = rnn.step_input(x_emb)
398
                # create prev memory parameter, batch size comes from word
399 400
                prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word)
                hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu')
401 402 403
                # use hidden to update prev
                rnn.update_memory(prev, hidden)
                # mark hidden as output 
404
                rnn.step_output(hidden)
405
            # get StaticrNN final output
406
            result = rnn()
C
chengduo 已提交
407

408
    """
Y
Yu Yang 已提交
409 410 411 412
    BEFORE_RNN_BLOCK = 0
    IN_RNN_BLOCK = 1
    AFTER_RNN_BLOCK = 2

413
    def __init__(self, name=None):
414
        check_type(name, "name", (str, type(None)), "fluid.layers.StaticRNN")
415
        self.helper = LayerHelper("static_rnn", name=name)
Y
Yu Yang 已提交
416 417 418 419 420 421 422 423
        self.memories = {}  # memory map, from pre_mem.name --> MemoryLink
        self.inputs = []  # input variable list in current block
        self.outputs = []  # output variable list in parent block
        self.status = StaticRNN.BEFORE_RNN_BLOCK  # status flag.
        # sequence length, since it is a static RNN, sequence length are fixed.
        self.seq_len = None

    def step(self):
C
chengduo 已提交
424
        """
425 426
        Define operators in each step. step is used in :code:`with` block, OP in :code:`with` block
        will be executed sequence_len times (sequence_len is the length of input)
C
chengduo 已提交
427
        """
Y
Yang Yang 已提交
428
        return BlockGuardWithCompletion(self)
Y
Yu Yang 已提交
429 430 431 432 433

    def _assert_in_rnn_block_(self, method):
        if self.status != StaticRNN.IN_RNN_BLOCK:
            raise ValueError("You must invoke {0} in rnn block".format(method))

434 435 436 437 438 439 440
    def memory(self,
               init=None,
               shape=None,
               batch_ref=None,
               init_value=0.0,
               init_batch_dim_idx=0,
               ref_batch_dim_idx=1):
441
        """
C
chengduo 已提交
442 443 444
        Create a memory variable for static rnn.
        If the :code:`init` is not None, :code:`memory` will be initialized by
        this Variable. If the :code:`init` is None, :code:`shape` and :code:`batch_ref`
445 446
        must be set, and this function will create a new variable with shape and batch_ref
        to initialize :code:`init` Variable.
C
chengduo 已提交
447

448
        Args:
449
            init(Variable, optional): Tensor used to init memory. If it is not set,
C
chengduo 已提交
450 451
                :code:`shape` and :code:`batch_ref` must be provided.
                Default: None.
452 453 454 455 456 457 458
            shape(list|tuple): When :code:`init` is None use this arg to initialize memory shape.
            NOTE the shape does not contain batch_size. Default: None.
            batch_ref(Variable, optional): When :code:`init` is None, memory's batch size will
            be set as batch_ref's ref_batch_dim_idx value. Default: None.
            init_value(float, optional): When :code:`init` is None, used to init memory's value. Default: 0.0.
            init_batch_dim_idx(int, optional): the batch_size axis of the :code:`init` Variable. Default: 0.
            ref_batch_dim_idx(int, optional): the batch_size axis of the :code:`batch_ref` Variable. Default: 1.
C
chengduo 已提交
459 460

        Returns:
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
            Variable: The memory variable.

        Examples 1:
            .. code-block:: python

            	import paddle.fluid as fluid
            	import paddle.fluid.layers as layers

            	vocab_size, hidden_size=10000, 200
            	x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64')
            	# create word sequence
            	x_emb = layers.embedding(
                	input=x,
                	size=[vocab_size, hidden_size],
                	dtype='float32',
                	is_sparse=False)
            	# transform batch size to dim 1
            	x_emb = layers.transpose(x_emb, perm=[1, 0, 2])

            	rnn = fluid.layers.StaticRNN()
            	with rnn.step():
                	# mark created x_emb as input, each step process a word
                	word = rnn.step_input(x_emb)
                	# create prev memory parameter, batch size comes from word
                	prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word)
                	hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu')
                	# use hidden to update prev
                	rnn.update_memory(prev, hidden)


        Examples 2:
492 493
            .. code-block:: python

494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
            	import paddle.fluid as fluid
            	import paddle.fluid.layers as layers
            	vocab_size, hidden_size=10000, 200
            	x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64')
            	# create word sequence
            	x_emb = layers.embedding(
                	input=x,
                	size=[vocab_size, hidden_size],
                	dtype='float32',
                	is_sparse=False)
            	# transform batch size to dim 1
            	x_emb = layers.transpose(x_emb, perm=[1, 0, 2])
            	boot_memory = fluid.layers.data(name='boot', shape=[hidden_size], dtype='float32', lod_level=1)
            	rnn = fluid.layers.StaticRNN()
            	with rnn.step():
            		# mark created x_emb as input, each step process a word
            		word = rnn.step_input(x_emb)
            		# init memory
            		prev = rnn.memory(init=boot_memory)
            		hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu')
            		# update hidden with prev
            		rnn.update_memory(prev, hidden)

517
        """
Y
Yu Yang 已提交
518
        self._assert_in_rnn_block_('memory')
519 520 521 522 523 524
        check_type(init, "init", (Variable, type(None)),
                   "fluid.layers.StaticRNN.memory")
        check_type(shape, "shape", (list, tuple, type(None)),
                   "fluid.layers.StaticRNN.memory")
        check_type(batch_ref, "batch_ref", (Variable, type(None)),
                   "fluid.layers.StaticRNN.memory")
Y
Yu Yang 已提交
525
        if init is None:
526
            if shape is None or batch_ref is None:
Y
Yu Yang 已提交
527
                raise ValueError(
528
                    "if init is None, memory at least need shape and batch_ref")
529
            parent_block = self._parent_block()
530
            var_name = unique_name.generate_with_ignorable_key("@".join(
Y
Yu Yang 已提交
531
                [self.helper.name, "memory_boot"]))
Y
Yu Yang 已提交
532
            boot_var = parent_block.create_var(
533 534
                name=var_name,
                shape=shape,
F
fengjiayi 已提交
535
                dtype=batch_ref.dtype,
536
                persistable=False)
Y
Yu Yang 已提交
537 538

            parent_block.append_op(
539 540
                type="fill_constant_batch_size_like",
                inputs={'Input': [batch_ref]},
Y
Yu Yang 已提交
541 542 543
                outputs={'Out': [boot_var]},
                attrs={
                    'value': init_value,
544
                    'shape': boot_var.shape,
F
fengjiayi 已提交
545
                    'dtype': boot_var.dtype,
546 547
                    'input_dim_idx': ref_batch_dim_idx,
                    'output_dim_idx': init_batch_dim_idx
Y
Yu Yang 已提交
548 549 550 551 552
                })

            return self.memory(init=boot_var)
        else:
            pre_mem = self.helper.create_variable(
553 554
                name=unique_name.generate_with_ignorable_key("@".join(
                    [self.helper.name, "mem"])),
F
fengjiayi 已提交
555
                dtype=init.dtype,
Y
Yu Yang 已提交
556 557 558 559 560 561
                shape=init.shape)
            self.memories[pre_mem.name] = StaticRNNMemoryLink(
                init=init, pre_mem=pre_mem)
            return pre_mem

    def step_input(self, x):
C
chengduo 已提交
562 563 564 565 566 567 568 569
        """
        Mark a sequence as a StaticRNN input.

        Args:
            x(Variable): The input sequence, the shape of x
                should be [seq_len, ...].

        Returns:
570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598
            Variable: The current time step data in the input sequence.

        Examples:
            .. code-block:: python

            	import paddle.fluid as fluid
            	import paddle.fluid.layers as layers

            	vocab_size, hidden_size=10000, 200
            	x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64')
            	# create word sequence
            	x_emb = layers.embedding(
                	input=x,
                	size=[vocab_size, hidden_size],
                	dtype='float32',
                	is_sparse=False)
            	# transform batch size to dim 1
            	x_emb = layers.transpose(x_emb, perm=[1, 0, 2])

            	rnn = fluid.layers.StaticRNN()
            	with rnn.step():
                	# mark created x_emb as input, each step process a word
                	word = rnn.step_input(x_emb)
                	# create prev memory parameter, batch size comes from word
                	prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word)
                	hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu')
                	# use hidden to update prev
                	rnn.update_memory(prev, hidden)

C
chengduo 已提交
599
        """
Y
Yu Yang 已提交
600
        self._assert_in_rnn_block_('step_input')
601
        check_type(x, "x", Variable, "fluid.layers.StaticRNN.step_input")
Y
Yu Yang 已提交
602
        if self.seq_len is None:
Y
Yu Yang 已提交
603
            self.seq_len = x.shape[0]
604
        elif x.shape[0] != -1 and self.seq_len != x.shape[0]:
Y
Yu Yang 已提交
605 606 607
            raise ValueError("Static RNN only take fix seq_len input")

        ipt = self.helper.create_variable(
F
fengjiayi 已提交
608
            name=x.name, dtype=x.dtype, shape=list(x.shape[1:]), type=x.type)
Y
Yu Yang 已提交
609 610 611 612
        self.inputs.append(ipt)
        return ipt

    def step_output(self, o):
C
chengduo 已提交
613 614 615 616 617 618 619 620
        """
        Mark a sequence as a StaticRNN output.

        Args:
            o(Variable): The output sequence.

        Returns:
            None.
621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651

        Examples:
            .. code-block:: python

            	import paddle.fluid as fluid
            	import paddle.fluid.layers as layers

            	vocab_size, hidden_size=10000, 200
            	x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64')
            	# create word sequence
            	x_emb = layers.embedding(
                	input=x,
                	size=[vocab_size, hidden_size],
               		dtype='float32',
                	is_sparse=False)
            	# transform batch size to dim 1
            	x_emb = layers.transpose(x_emb, perm=[1, 0, 2])

            	rnn = fluid.layers.StaticRNN()
            	with rnn.step():
                	# mark created x_emb as input, each step process a word
               		word = rnn.step_input(x_emb)
                	# create prev memory parameter, batch size comes from word
                	prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word)
                	hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu')
                	# use hidden to update prev
                	rnn.update_memory(prev, hidden)
                	rnn.step_output(hidden)

            	result = rnn()

C
chengduo 已提交
652
        """
Y
Yu Yang 已提交
653
        self._assert_in_rnn_block_('step_output')
654
        check_type(o, "o", Variable, "fluid.layers.StaticRNN.step_output")
Y
Yu Yang 已提交
655

X
Xin Pan 已提交
656
        tmp_o = self.helper.create_variable_for_type_inference(dtype=o.dtype)
Y
Yu Yang 已提交
657 658 659 660
        self.helper.append_op(
            type='rnn_memory_helper',
            inputs={'X': [o]},
            outputs={'Out': tmp_o},
F
fengjiayi 已提交
661
            attrs={'dtype': o.dtype})
Y
Yu Yang 已提交
662

663
        out_var = self._parent_block().create_var(
Y
Yu Yang 已提交
664 665
            name=tmp_o.name,
            shape=[self.seq_len] + list(tmp_o.shape),
F
fengjiayi 已提交
666
            dtype=tmp_o.dtype)
Y
Yu Yang 已提交
667 668 669 670

        self.outputs.append(out_var)

    def output(self, *outputs):
C
chengduo 已提交
671 672 673 674
        """
        Mark the StaticRNN output variables.

        Args:
675
            outputs: The output Tensor, can mark multiple variables as output
C
chengduo 已提交
676 677 678

        Returns:
            None
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709

        Examples:
            .. code-block:: python

            	import paddle.fluid as fluid
            	import paddle.fluid.layers as layers

            	vocab_size, hidden_size=10000, 200
            	x = fluid.data(name="x", shape=[None, 1, 1], dtype='int64')
            	# create word sequence
            	x_emb = layers.embedding(
                	input=x,
                	size=[vocab_size, hidden_size],
                	dtype='float32',
                	is_sparse=False)
            	# transform batch size to dim 1
            	x_emb = layers.transpose(x_emb, perm=[1, 0, 2])

            	rnn = fluid.layers.StaticRNN()
            	with rnn.step():
                	# mark created x_emb as input, each step process a word
                	word = rnn.step_input(x_emb)
                	# create prev memory parameter, batch size comes from word
                	prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word)
                	hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu')
                	# use hidden to update prev
                	rnn.update_memory(prev, hidden)
                	# mark each step's hidden and word as output
                	rnn.output(hidden, word)

            	result = rnn()
C
chengduo 已提交
710
        """
Y
Yu Yang 已提交
711 712 713 714
        for each in outputs:
            self.step_output(each)

    def update_memory(self, mem, var):
C
chengduo 已提交
715
        """
716
        Update the memory from :code:`mem` to :code:`var`.
C
chengduo 已提交
717 718 719

        Args:
            mem(Variable): the memory variable.
720
            var(Variable): the plain variable generated in RNN block, used to update memory.
T
tianshuo78520a 已提交
721
                           var and mem should have same dims and data type.
C
chengduo 已提交
722 723 724

        Returns:
            None
725

C
chengduo 已提交
726
        """
727 728
        check_type(mem, "mem", Variable, "fluid.layers.StaticRNN.update_memory")
        check_type(var, "var", Variable, "fluid.layers.StaticRNN.update_memory")
Y
Yu Yang 已提交
729 730
        self.memories[mem.name].mem = var

731
    def _parent_block(self):
732
        prog = self.helper.main_program
Y
Yu Yang 已提交
733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
        parent_idx = prog.current_block().parent_idx
        assert parent_idx >= 0
        parent_block = prog.block(parent_idx)
        return parent_block

    def __call__(self, *args, **kwargs):
        if self.status != StaticRNN.AFTER_RNN_BLOCK:
            raise ValueError("RNN output can only be retrieved after rnn block")
        if len(self.outputs) == 0:
            raise ValueError("RNN has no output")
        elif len(self.outputs) == 1:
            return self.outputs[0]
        else:
            return self.outputs

748
    def _complete_op(self):
749 750
        main_program = self.helper.main_program
        rnn_block = main_program.current_block()
751
        parent_block = self._parent_block()
Y
Yu Yang 已提交
752 753 754 755 756 757 758 759 760 761 762 763 764 765

        local_inputs = set()

        for op in rnn_block.ops:
            assert isinstance(op, Operator)
            for oname in op.output_names:
                for out_var_name in op.output(oname):
                    local_inputs.add(out_var_name)

        for var in self.inputs:
            local_inputs.add(var.name)
        for m in self.memories:
            local_inputs.add(m)

C
chengduo 已提交
766 767 768
        # NOTE(zcd): the params have two categories of variables.
        #   - the variables that are the out of StaticRnn.
        #   - the variables that are the parameters of some layers, for example, conv2d.
Y
Yu Yang 已提交
769 770 771 772 773 774 775 776
        params = list()
        for op in rnn_block.ops:
            assert isinstance(op, Operator)
            for iname in op.input_names:
                for in_var_name in op.input(iname):
                    if in_var_name not in local_inputs:
                        params.append(in_var_name)

777
        parameters = [parent_block.var(name) for name in set(params)]
Y
Yu Yang 已提交
778 779 780 781 782 783 784

        step_scope = parent_block.create_var(
            type=core.VarDesc.VarType.STEP_SCOPES)

        inlinks = [parent_block.var(i.name) for i in self.inputs]
        outlinks = self.outputs

C
chengduo 已提交
785
        # NOTE(zcd): the states maybe empty in some case.
Y
Yu Yang 已提交
786 787 788
        boot_memories = []
        pre_memories = []
        memories = []
M
minqiyang 已提交
789
        for _, mem in six.iteritems(self.memories):
Y
Yu Yang 已提交
790 791
            boot_memories.append(mem.init)
            pre_memories.append(mem.pre_mem.name)
C
chengduo 已提交
792 793
            assert mem.mem is not None, "%s should be updated in every step." % (
                mem.init.name)
Y
Yu Yang 已提交
794 795
            mem_var = rnn_block.var(mem.mem.name)
            assert isinstance(mem_var, Variable)
X
Xin Pan 已提交
796 797
            new_mem = self.helper.create_variable_for_type_inference(
                dtype=mem_var.dtype)
Y
Yu Yang 已提交
798 799 800 801
            rnn_block.append_op(
                type='rnn_memory_helper',
                inputs={'X': [mem_var]},
                outputs={'Out': [new_mem]},
F
fengjiayi 已提交
802
                attrs={'dtype': mem_var.dtype})
Y
Yu Yang 已提交
803 804 805 806 807 808 809 810 811 812 813 814 815

            memories.append(new_mem.name)

        parent_block.append_op(
            type='recurrent',
            inputs={
                'inputs': inlinks,
                'initial_states': boot_memories,
                'parameters': parameters
            },
            outputs={'outputs': outlinks,
                     'step_scopes': [step_scope]},
            attrs={
C
chengduo 已提交
816
                'has_states': len(pre_memories) > 0,
Y
Yu Yang 已提交
817 818
                'ex_states': pre_memories,
                'states': memories,
819
                'sub_block': rnn_block
Y
Yu Yang 已提交
820
            })
Y
Yu Yang 已提交
821 822


Y
Yang Yang(Tony) 已提交
823 824 825 826 827 828 829 830 831 832 833 834 835 836 837
class WhileGuard(BlockGuard):
    def __init__(self, while_op):
        if not isinstance(while_op, While):
            raise TypeError("WhileGuard takes a while op")
        super(WhileGuard, self).__init__(while_op.helper.main_program)
        self.while_op = while_op

    def __enter__(self):
        self.while_op.status = While.IN_WHILE_BLOCK
        return super(WhileGuard, self).__enter__()

    def __exit__(self, exc_type, exc_val, exc_tb):
        if exc_type is not None:
            return False
        self.while_op.status = While.AFTER_WHILE_BLOCK
838
        self.while_op._complete()
Y
Yang Yang(Tony) 已提交
839 840 841 842
        return super(WhileGuard, self).__exit__(exc_type, exc_val, exc_tb)


class While(object):
X
Xin Pan 已提交
843
    """
844
    while loop control flow. Repeat while body until cond is False.
X
Xin Pan 已提交
845

846 847 848 849
    Note:
        A new OP :ref:`api_fluid_layers_while_loop` is highly recommended instead of ``While`` if the shape of parameter ``cond`` is [1].
        OP :ref:`api_fluid_layers_while_loop` is easier to use and is called with less code but does the same thing as ``While`` .

X
Xin Pan 已提交
850
    Args:
851
        cond(Variable): A Tensor whose data type is bool controlling whether to continue looping.
G
guofei 已提交
852
        is_test(bool, optional): A flag indicating whether execution is in test phase. Default value is False.
853
        name(str, optional): The default value is None.  Normally there is no need for user to set this property.  For more information, please refer to :ref:`api_guide_Name` .
X
Xin Pan 已提交
854 855 856

    Examples:
          .. code-block:: python
857 858
            
            import paddle.fluid as fluid
859 860 861 862 863
            import numpy as np

            i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0)           # loop counter

            loop_len = fluid.layers.fill_constant(shape=[1],dtype='int64', value=10)    # loop length
864

865
            cond = fluid.layers.less_than(x=i, y=loop_len)              
866
            while_op = fluid.layers.While(cond=cond)
867
            with while_op.block():  
868
                i = fluid.layers.increment(x=i, value=1, in_place=True)
869 870 871 872 873 874 875
                fluid.layers.less_than(x=i, y=loop_len, cond=cond)      

            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(fluid.default_startup_program())

            res = exe.run(fluid.default_main_program(), feed={}, fetch_list=[i])
            print(res) # [array([10])]           
X
Xin Pan 已提交
876 877
    """

Y
Yang Yang(Tony) 已提交
878 879 880 881
    BEFORE_WHILE_BLOCK = 0
    IN_WHILE_BLOCK = 1
    AFTER_WHILE_BLOCK = 2

C
chengduo 已提交
882
    def __init__(self, cond, is_test=False, name=None):
883
        self.helper = LayerHelper("while", name=name)
Y
Yang Yang(Tony) 已提交
884 885 886 887
        self.status = While.BEFORE_WHILE_BLOCK
        if not isinstance(cond, Variable):
            raise TypeError("condition should be a variable")
        assert isinstance(cond, Variable)
888
        if cond.dtype != core.VarDesc.VarType.BOOL:
889
            raise TypeError("condition should be a boolean variable")
Y
Yang Yang(Tony) 已提交
890
        if reduce(lambda a, b: a * b, cond.shape, 1) != 1:
891 892 893
            raise TypeError(
                "condition expected shape as [], but given shape as {0}.".
                format(list(cond.shape)))
Y
Yang Yang(Tony) 已提交
894
        self.cond_var = cond
C
chengduo 已提交
895
        self.is_test = is_test
Y
Yang Yang(Tony) 已提交
896 897 898 899

    def block(self):
        return WhileGuard(self)

900
    def _complete(self):
Y
Yang Yang(Tony) 已提交
901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
        main_program = self.helper.main_program
        while_block = main_program.current_block()
        parent_block = main_program.block(main_program.current_block()
                                          .parent_idx)

        inner_outputs = {self.cond_var.name}
        x_name_list = set()
        for op in while_block.ops:
            for iname in op.input_names:
                for in_var_name in op.input(iname):
                    if in_var_name not in inner_outputs:
                        x_name_list.add(in_var_name)

            for oname in op.output_names:
                for out_var_name in op.output(oname):
                    inner_outputs.add(out_var_name)

        out_vars = []
        for inner_out_name in inner_outputs:
X
Xin Pan 已提交
920 921 922
            inner_var = parent_block._find_var_recursive(inner_out_name)
            if inner_var:
                out_vars.append(inner_var)
Y
Yang Yang(Tony) 已提交
923 924 925 926 927 928 929

        step_scope = parent_block.create_var(
            type=core.VarDesc.VarType.STEP_SCOPES)

        parent_block.append_op(
            type='while',
            inputs={
W
Wu Yi 已提交
930 931 932 933
                'X': [
                    parent_block._var_recursive(x_name)
                    for x_name in x_name_list
                ],
Y
Yang Yang(Tony) 已提交
934 935 936 937
                'Condition': [self.cond_var]
            },
            outputs={'Out': out_vars,
                     'StepScopes': [step_scope]},
C
chengduo 已提交
938 939
            attrs={'sub_block': while_block,
                   "is_test": self.is_test})
Y
Yang Yang(Tony) 已提交
940 941


G
guofei 已提交
942
def while_loop(cond, body, loop_vars, is_test=False, name=None):
G
guofei 已提交
943 944 945 946
    """
    while_loop is one of the control flows. Repeats while_loop `body` until `cond` returns False.

    Args:
947 948 949 950 951
        cond(Callable): A callable returning a boolean tensor controlling whether to continue looping. And ``cond`` takes
	    as many arguments as ``loop_vars`` .
        body(Callable): A callable returning a tuple or list of tensors or LoDTensorArrays of the same arity
            (length and structure) and types as ``loops_vars`` . And ``body`` takes as many arguments as ``loop_vars`` .
        loop_vars(list|tuple): A list or tuple of tensors or LoDTensorArrays that is passed to both ``cond`` and ``body`` .
G
guofei 已提交
952
        is_test(bool, optional): A flag indicating whether execution is in test phase. Default value is False.
G
guofei 已提交
953 954 955 956
        name(str, optional): Normally there is no need for users to set this property. For more information, please
            refer to :ref:`api_guide_Name`. Default is None.
    
    Returns:
957
        A list or tuple of tensors or LoDTensorArrays which returned by ``body`` .
G
guofei 已提交
958 959 960 961 962 963 964 965 966 967 968 969
    
    Returen type:
        list(Variable)|tuple(Variable).

    Raises:
        TypeError: If the type of ``cond`` is not callable.
        TypeError: If the type of ``body`` is not callable.
        TypeError: If the type of ``loop_vars`` is not list or tuple.
        TypeError: If the type of ``cond`` returns is not Variable.
        TypeError: If the type of ``cond`` returns is not a boolean variable.
        TypeError: If the shape of ``cond`` returns is not equals 1.
        ValueError: If the ``var_loops`` is empty.
970
        ValueError: If the length or type of ``body`` returns is not same as ``loop_vars``.
G
guofei 已提交
971 972 973 974 975 976 977

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            import paddle.fluid.layers as layers

978 979
            def cond(i, ten):
                return i < ten
G
guofei 已提交
980

981 982 983
            def body(i, ten):
                i = i + 1
                return [i, ten]
G
guofei 已提交
984 985 986 987 988 989

            main_program = fluid.default_main_program()
            startup_program = fluid.default_startup_program()
            with fluid.program_guard(main_program, startup_program):
                i = layers.fill_constant(shape=[1], dtype='int64', value=0)     # loop counter
                ten = layers.fill_constant(shape=[1], dtype='int64', value=10)  # loop length
990
                i, ten = layers.while_loop(cond, body, [i, ten])
G
guofei 已提交
991 992
                
                exe = fluid.Executor(fluid.CPUPlace())
993
                res = exe.run(main_program, feed={}, fetch_list=[i])
G
guofei 已提交
994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
                print(res) # [array([10])]
    """
    helper = LayerHelper('while_loop', **locals())

    if not callable(cond):
        raise TypeError("cond in while_loop should be callable")
    if not callable(body):
        raise TypeError("body in while_loop should be callable")
    if not isinstance(loop_vars, (list, tuple)):
        raise TypeError("loop_vars in while_loop should be a list or tuple")
    if len(loop_vars) == 0:
        raise ValueError("loop_vars in while_loop should not be empty")

    pre_cond = cond(*loop_vars)
    if not isinstance(pre_cond, Variable):
        raise TypeError("cond in while_loop should return a variable")
    if pre_cond.dtype != core.VarDesc.VarType.BOOL:
        raise TypeError("cond in while_loop should return a boolean variable")
    if reduce(lambda a, b: a * b, pre_cond.shape, 1) != 1:
        raise TypeError(
            "the shape of the variable returned by cond should be [],"
            "but given shape as {0}.".format(list(pre_cond.shape)))

1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
    if in_dygraph_mode():
        now_cond = pre_cond.numpy()[0]
        while (now_cond):
            output_vars = body(*loop_vars)
            if not isinstance(output_vars, (list, tuple)):
                output_vars = [output_vars]
            if len(output_vars) != len(loop_vars):
                raise ValueError(
                    "body in while_loop should return the same arity "
                    "(length and structure) and types as loop_vars")
            now_cond = cond(*output_vars).numpy()[0]
            loop_vars = output_vars
        return loop_vars

G
guofei 已提交
1031
    while_loop_block = While(pre_cond, is_test, name)
1032
    has_mutable_vars_in_loop = hold_mutable_vars(loop_vars)
G
guofei 已提交
1033
    with while_loop_block.block():
1034 1035 1036 1037 1038 1039 1040 1041 1042
        # If a variable with mutable type is included in loop_vars, like `dict/list`,
        # modifying it in the body function will cause origin variable to be modified
        # synchronously. This will raise an assignment error out of while block.
        # Here we make a copy of the mutable vars to avoid this problem.
        if has_mutable_vars_in_loop:
            new_loop_vars = copy_mutable_vars(loop_vars)
            output_vars = body(*new_loop_vars)
        else:
            output_vars = body(*loop_vars)
1043 1044
        if not isinstance(output_vars, (list, tuple)):
            output_vars = [output_vars]
1045 1046 1047
        try:
            assert_same_structure(output_vars, loop_vars, check_types=False)
        except ValueError as e:
1048
            raise ValueError("body in while_loop should return the same arity "
1049 1050
                             "(length and structure) as loop_vars: {0}".format(
                                 e))
1051
        now_cond = cond(*output_vars)
1052
        map_structure(assign, output_vars, loop_vars)
G
guofei 已提交
1053 1054 1055 1056
        assign(now_cond, pre_cond)
    return loop_vars


1057
def lod_rank_table(x, level=0):
1058 1059
    """
    LoD Rank Table Operator. Given an input variable **x** and a level number
Y
yangyaming 已提交
1060 1061
    of LoD, this layer creates a LodRankTable object. A LoDRankTable object
    contains a list of bi-element tuples. Each tuple consists of an index and
1062
    a length, both of which are int type. Refering to specified level of LoD,
T
tianshuo78520a 已提交
1063
    the index is the sequence index number and the length represents the
Y
yangyaming 已提交
1064 1065
    sequence length. Please note that the list is ranked in descending order by
    the length. The following is an example:
Y
yangyaming 已提交
1066 1067 1068 1069

        .. code-block:: text

            x is a LoDTensor:
1070 1071
                x.lod = [[2,                1],
                         [5,             1, 1]]
Y
yangyaming 已提交
1072 1073
                x.data = [a, b, c, d, e, f, g]

Y
yangyaming 已提交
1074 1075 1076
            1. set level to 0:
                Create lod rank table:
                    lod_rank_table_obj = lod_rank_table(x, level=0)
Y
yangyaming 已提交
1077

Y
yangyaming 已提交
1078 1079 1080 1081 1082 1083 1084 1085 1086
                Get:
                    lod_rank_table_obj.items() = [(0, 2), (1, 1)]

            2. set level to 1:
                Create lod rank table:
                    lod_rank_table_obj = lod_rank_table(x, level=1)

                Get:
                    lod_rank_table_obj.items() = [(0, 5), (1, 1), (2, 1)]
Y
yangyaming 已提交
1087 1088 1089 1090

    Args:
        x (Variable): Input variable, a LoDTensor based which to create the lod
            rank table.
Y
yangyaming 已提交
1091 1092
        level (int): Specify the LoD level, on which to create the lod rank
            table.
Y
yangyaming 已提交
1093 1094 1095 1096 1097 1098 1099

    Returns:
        Variable: The created LoDRankTable object.

    Examples:
        .. code-block:: python

1100
            import paddle.fluid as fluid
Y
yangyaming 已提交
1101
            x = fluid.layers.data(name='x', shape=[10],
1102
                                  dtype='float32', lod_level=1)
Y
yangyaming 已提交
1103
            out = layers.lod_rank_table(x=x, level=0)
1104
    """
Y
Yu Yang 已提交
1105 1106 1107
    helper = LayerHelper("lod_rank_table", **locals())
    table = helper.create_variable(
        type=core.VarDesc.VarType.LOD_RANK_TABLE,
Y
Yu Yang 已提交
1108
        name=unique_name.generate("lod_rank_table"))
Y
Yu Yang 已提交
1109 1110 1111 1112 1113 1114
    helper.append_op(
        type='lod_rank_table',
        inputs={'X': x},
        outputs={'Out': table},
        attrs={'level': level})
    return table
Y
Yu Yang 已提交
1115 1116


Y
yuyang18 已提交
1117
@templatedoc()
1118
def max_sequence_len(rank_table):
Y
yuyang18 已提交
1119 1120 1121 1122 1123 1124 1125 1126
    """
    ${comment}

    >>> import paddle.fluid as fluid
    >>> x = fluid.layers.data(name='x', shape=[10], dtype='float32',
    >>>                       lod_level=1)
    >>> rank_table = layers.lod_rank_table(x=x, level=0)
    >>> max_seq_len = layers.max_sequence_len(rank_table)
Y
yangyaming 已提交
1127 1128

    Args:
Y
yuyang18 已提交
1129
        rank_table(${rank_table_type}): ${rank_table_comment}.
Y
yangyaming 已提交
1130 1131

    Returns:
Y
yuyang18 已提交
1132
        ${out_comment}.
F
fengjiayi 已提交
1133 1134
    """
    helper = LayerHelper("max_seqence_len", **locals())
X
Xin Pan 已提交
1135
    res = helper.create_variable_for_type_inference(dtype="int64")
F
fengjiayi 已提交
1136 1137 1138 1139 1140 1141 1142
    helper.append_op(
        type="max_sequence_len",
        inputs={"RankTable": rank_table},
        outputs={"Out": res})
    return res


1143
def lod_tensor_to_array(x, table):
1144
    """
F
fengjiayi 已提交
1145 1146
    Convert a LoDTensor to a LoDTensorArray.

1147 1148 1149 1150 1151
    This function split a LoDTesnor to a LoDTensorArray according to its LoD
    information. LoDTensorArray is an alias of C++ std::vector<LoDTensor> in
    PaddlePaddle. The generated LoDTensorArray of this function can be further read
    or written by `read_from_array()` and `write_to_array()` operators. However,
    this function is generally an internal component of PaddlePaddle `DynamicRNN`.
F
fengjiayi 已提交
1152
    Users should not use it directly.
1153 1154

    Args:
F
fengjiayi 已提交
1155
        x (Variable|list): The LoDTensor to be converted to a LoDTensorArray.
1156 1157
        table (ParamAttr|list): The variable that stores the level of lod
                                which is ordered by sequence length in
1158
                                descending order. It is generally generated
F
fengjiayi 已提交
1159
                                by `layers.lod_rank_table()` API.
1160 1161

    Returns:
F
fengjiayi 已提交
1162
        Variable: The LoDTensorArray that has been converted from the input tensor.
1163 1164 1165 1166

    Examples:
        .. code-block:: python

1167
          import paddle.fluid as fluid
1168 1169 1170
          x = fluid.layers.data(name='x', shape=[10])
          table = fluid.layers.lod_rank_table(x, level=0)
          array = fluid.layers.lod_tensor_to_array(x, table)
1171
    """
1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
    check_type(x, 'x', (Variable, list), 'lod_tensor_to_array')
    if isinstance(x, (list)):
        for i, input_x in enumerate(x):
            check_type(input_x, 'input[' + str(i) + ']', Variable,
                       'lod_tensor_to_array')
    check_type(table, 'table', (Variable, list), 'lod_tensor_to_array')
    if isinstance(table, (list)):
        for i, table_x in enumerate(table):
            check_type(table_x, 'table[' + str(i) + ']', Variable,
                       'lod_tensor_to_array')
1182 1183
    helper = LayerHelper("lod_tensor_to_array", **locals())
    array = helper.create_variable(
Y
Yu Yang 已提交
1184
        name=unique_name.generate("lod_tensor_to_array"),
1185
        type=core.VarDesc.VarType.LOD_TENSOR_ARRAY,
F
fengjiayi 已提交
1186
        dtype=x.dtype)
1187 1188 1189 1190 1191 1192 1193 1194
    helper.append_op(
        type='lod_tensor_to_array',
        inputs={'X': x,
                'RankTable': table},
        outputs={'Out': array})
    return array


1195
def array_to_lod_tensor(x, table):
1196
    """Convert a LoD_Tensor_Aarry to an LoDTensor.
1197 1198

    Args:
1199
        x (Variable|list): The lod tensor array to be converted to a tensor.
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210
        table (ParamAttr|list): The variable that stores the level of lod
                                which is ordered by sequence length in
                                descending order.

    Returns:
        Variable: The variable of type tensor that has been converted
                  from an array.

    Examples:
        .. code-block:: python

1211
          import paddle.fluid as fluid
1212 1213 1214 1215
          x = fluid.layers.data(name='x', shape=[10])
          table = fluid.layers.lod_rank_table(x, level=0)
          array = fluid.layers.lod_tensor_to_array(x, table)
          lod_tensor = fluid.layers.array_to_lod_tensor(array, table)
1216
    """
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227
    check_type(x, 'x', (Variable, list), 'array_to_lod_tensor')
    if isinstance(x, (list)):
        for i, input_x in enumerate(x):
            check_type(input_x, 'input[' + str(i) + ']', Variable,
                       'array_to_lod_tensor')
    check_type(table, 'table', (Variable, list), 'array_to_lod_tensor')
    if isinstance(table, (list)):
        for i, table_x in enumerate(table):
            check_type(table_x, 'table[' + str(i) + ']', Variable,
                       'array_to_lod_tensor')

1228
    helper = LayerHelper("array_to_lod_tensor", **locals())
X
Xin Pan 已提交
1229
    tmp = helper.create_variable_for_type_inference(dtype=x.dtype)
1230 1231 1232 1233 1234 1235 1236 1237
    helper.append_op(
        type="array_to_lod_tensor",
        inputs={'X': x,
                'RankTable': table},
        outputs={'Out': tmp})
    return tmp


1238
def increment(x, value=1.0, in_place=True):
1239
    """
1240 1241
    The OP is usually used for control flow to increment the data of :attr:`x` by an amount :attr:`value`.
    Notice that the number of elements in :attr:`x` must be equal to 1.
1242

1243
    Parameters:
T
tianshuo78520a 已提交
1244
        x (Variable): A tensor that must always contain only one element, its data type supports
1245 1246 1247
            float32, float64, int32 and int64.
        value (float, optional): The amount to increment the data of :attr:`x`. Default: 1.0.
        in_place (bool, optional): Whether the OP should be performed in-place. Default: True.
1248 1249

    Returns:
1250
        Variable: The elementwise-incremented tensor with the same shape and data type as :attr:`x`.
1251 1252 1253 1254

    Examples:
        .. code-block:: python

1255
          import paddle.fluid as fluid
1256 1257
          counter = fluid.layers.zeros(shape=[1], dtype='float32') # [0.]
          fluid.layers.increment(counter) # [1.]
1258
    """
1259 1260
    check_variable_and_dtype(x, 'x', ['float32', 'float64', 'int32', 'int64'],
                             'increment')
Y
Yu Yang 已提交
1261
    helper = LayerHelper("increment", **locals())
Y
Yang Yang(Tony) 已提交
1262
    if not in_place:
X
Xin Pan 已提交
1263
        out = helper.create_variable_for_type_inference(dtype=x.dtype)
Y
Yang Yang(Tony) 已提交
1264 1265
    else:
        out = x
Y
Yu Yang 已提交
1266 1267 1268
    helper.append_op(
        type='increment',
        inputs={'X': [x]},
Y
Yang Yu 已提交
1269
        outputs={'Out': [out]},
1270
        attrs={'step': float(value)})
Y
Yang Yu 已提交
1271
    return out
Y
Yu Yang 已提交
1272 1273


1274
def array_write(x, i, array=None):
1275
    """
1276 1277 1278 1279
    This OP writes the input ``x`` into the i-th position of the ``array``
    :ref:`api_fluid_LoDTensorArray` and returns the modified array.
    If ``array`` is none, a new LoDTensorArray will be created and returned.
    This OP is often used together with :ref:`api_fluid_layers_array_read` OP.
1280 1281

    Args:
1282 1283 1284 1285 1286 1287 1288
        x (Variable): The input data to be written into array. It's multi-dimensional
            Tensor or LoDTensor. Data type: float32, float64, int32, int64.
        i (Variable): 1-D Tensor with shape [1], which represents the position into which
            ``x`` is written. Data type: int64.
        array (LoDTensorArray, optional): The LoDTensorArray into which ``x`` is written. 
            The default value is None, when a new LoDTensorArray will be created and returned 
            as a result.
1289

1290
    Returns:
1291
        Variable: The input ``array`` after ``x`` is written into.
1292 1293

    Examples:
D
dzhwinter 已提交
1294
        .. code-block:: python
1295

1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
            import paddle.fluid as fluid
            tmp = fluid.layers.fill_constant(shape=[3, 2], dtype='int64', value=5)
            i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10)
            # Write tmp into the position of arr with subscript 10 and return arr.
            arr = fluid.layers.array_write(tmp, i=i)

            # Now, arr is a LoDTensorArray with length 11. We can use array_read OP to read
            # the data at subscript 10 and print it out.
            item = fluid.layers.array_read(arr, i=i)
            input = fluid.layers.Print(item, message="The content of i-th LoDTensor:")
            main_program = fluid.default_main_program()
            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(main_program)

            # The printed result is:
            # 1570533133    The content of i-th LoDTensor:  The place is:CPUPlace
            # Tensor[array_read_0.tmp_0]
            #    shape: [3,2,]
            #    dtype: l
            #    data: 5,5,5,5,5,5,

            # the output is 2-D Tensor with shape [3,2], which is tmp above.
            # dtype is the corresponding C++ data type, which may vary in different environments.
            # Eg: if the data type of tensor is int64, then the corresponding C++ data type is int64_t, 
            #       so the dtype value is typeid(int64_t).Name(), which is 'x' on MacOS, 'l' on Linux, 
            #       and '__int64' on Windows. They both represent 64-bit integer variables.

1323
    """
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
    if in_dygraph_mode():
        assert isinstance(
            x, Variable
        ), "The input data 'x' in array_write must be Variable in dygraph mode"
        assert isinstance(
            i, Variable
        ), "The index 'i' in array_write must be Variable in dygraph mode"
        assert i.shape == [
            1
        ], "The shape of index 'i' should be [1] in dygraph mode"
        i = i.numpy()[0]
        if array is None:
            array = create_array(x.dtype)
        assert isinstance(
            array,
            list), "The 'array' in array_write must be a list in dygraph mode"
        assert i <= len(
            array
        ), "The index 'i' should not be greater than the length of 'array' in dygraph mode"
        if i < len(array):
            array[i] = x
        else:
            array.append(x)
        return array

1349 1350
    check_variable_and_dtype(i, 'i', ['int64'], 'array_write')
    check_type(x, 'x', (Variable), 'array_write')
Y
Yu Yang 已提交
1351
    helper = LayerHelper('array_write', **locals())
1352 1353 1354 1355 1356 1357
    if array is not None:
        if not isinstance(
                array,
                Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY:
            raise TypeError(
                "array should be tensor array vairable in array_write Op")
Y
Yu Yang 已提交
1358 1359 1360 1361
    if array is None:
        array = helper.create_variable(
            name="{0}.out".format(helper.name),
            type=core.VarDesc.VarType.LOD_TENSOR_ARRAY,
F
fengjiayi 已提交
1362
            dtype=x.dtype)
Y
Yu Yang 已提交
1363 1364 1365 1366 1367 1368 1369 1370
    helper.append_op(
        type='write_to_array',
        inputs={'X': [x],
                'I': [i]},
        outputs={'Out': [array]})
    return array


1371
def create_array(dtype):
1372
    """
1373 1374 1375 1376
    This OP creates an LOD_TENSOR_ARRAY. It is used as
    the input of :ref:`api_fluid_layers_array_read` and 
    :ref:`api_fluid_layers_array_write`. Also it can be used
    with  :ref:`api_fluid_layers_While` to create RNN network.
1377 1378

    Args:
1379 1380
        dtype (str): The data type of the elements in the lod_tensor_array.
                     Support data type: float32, float64, int32, int64.
1381 1382

    Returns:
1383
        Variable: The empty lod_tensor_array. The data type of elements in Tensor is ``dtype``.
1384 1385 1386 1387

    Examples:
        .. code-block:: python

1388
          import paddle.fluid as fluid
1389
          data = fluid.layers.create_array(dtype='float32') # Create a float32 LoDTensorArray.
1390 1391

    """
1392 1393 1394
    if in_dygraph_mode():
        return []

Y
Yang Yang(Tony) 已提交
1395 1396 1397 1398 1399 1400 1401
    helper = LayerHelper("array", **locals())
    return helper.create_variable(
        name="{0}.out".format(helper.name),
        type=core.VarDesc.VarType.LOD_TENSOR_ARRAY,
        dtype=dtype)


Y
yuyang18 已提交
1402
@templatedoc()
1403
def less_than(x, y, force_cpu=None, cond=None):
1404
    """
Y
yuyang18 已提交
1405
    ${comment}
1406 1407

    Args:
Y
yuyang18 已提交
1408 1409 1410
        x(${x_type}): ${x_comment}.
        y(${y_type}): ${y_comment}.
        force_cpu(${force_cpu_type}): ${force_cpu_comment}.
1411 1412 1413
        cond(Variable|None): Optional output variable to store the result of *less_than*

    Returns:
Y
yuyang18 已提交
1414
        ${out_comment}.
1415 1416 1417 1418

    Examples:
        .. code-block:: python

1419
          import paddle.fluid as fluid
W
Wilber 已提交
1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
          import numpy as np
  
          # Graph Organizing
          x = fluid.layers.data(name='x', shape=[2], dtype='float64')
          y = fluid.layers.data(name='y', shape=[2], dtype='float64')
          result = fluid.layers.less_than(x=x, y=y)
          # The comment lists another available method.
          # result = fluid.layers.fill_constant(shape=[2], dtype='float64', value=0)
          # fluid.layers.less_than(x=x, y=y, cond=result)
  
          # Create an executor using CPU as example
          exe = fluid.Executor(fluid.CPUPlace())
  
          # Execute
          x_i = np.array([[1, 2], [3, 4]]).astype(np.float64)
          y_i = np.array([[2, 2], [1, 3]]).astype(np.float64)
          result_value, = exe.run(fluid.default_main_program(), feed={'x':x_i, 'y':y_i}, fetch_list=[result])
          print(result_value) # [[True, False], [False, False]]
1438
    """
1439 1440 1441 1442 1443 1444 1445 1446 1447
    check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"],
                             "less_than")
    check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"],
                             "less_than")
    if cond is not None:
        check_type(cond, "cond", Variable, "less_than")
    if force_cpu != None:
        check_type(force_cpu, "force_cpu", bool, "less_than")

Y
Yang Yang(Tony) 已提交
1448 1449
    helper = LayerHelper("less_than", **locals())
    if cond is None:
X
Xin Pan 已提交
1450
        cond = helper.create_variable_for_type_inference(dtype='bool')
Y
Yang Yang(Tony) 已提交
1451 1452
        cond.stop_gradient = True

Y
yuyang18 已提交
1453 1454 1455 1456
    attrs = dict()
    if force_cpu is not None:
        attrs['force_cpu'] = force_cpu

Y
Yang Yang(Tony) 已提交
1457
    helper.append_op(
J
JiayiFeng 已提交
1458 1459 1460 1461
        type='less_than',
        inputs={'X': [x],
                'Y': [y]},
        outputs={'Out': [cond]},
Y
yuyang18 已提交
1462
        attrs=attrs)
Y
Yang Yang(Tony) 已提交
1463 1464 1465
    return cond


Z
zhoukunsheng 已提交
1466 1467 1468
@templatedoc()
def less_equal(x, y, cond=None):
    """
1469
    This OP returns the truth value of :math:`x <= y` elementwise, which is equivalent function to the overloaded operator `<=`.
Z
zhoukunsheng 已提交
1470 1471

    Args:
1472 1473 1474 1475 1476
        x(Variable): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. 
        y(Variable): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64.
        cond(Variable, optional): If is :attr:`None`, the op will create a variable as output tensor, the input shape and data type of \
            this tensor is the same as input :attr:`x`. If is not :attr:`None`, the op will set the variable as output tensor, the input shape \
            and data type of this tensor should be the same as input :attr:`x`. Default value is :attr:`None`.
Z
zhoukunsheng 已提交
1477 1478

    Returns:
1479
        Variable, the output data type is bool.: The tensor variable storing the output, the output shape is the same as input :attr:`x`.
Z
zhoukunsheng 已提交
1480 1481 1482 1483

    Examples:
        .. code-block:: python

1484
          import paddle.fluid as fluid
1485 1486 1487 1488 1489 1490
          import numpy as np
          label = fluid.layers.assign(np.array([1, 3], dtype='int32'))
          limit = fluid.layers.assign(np.array([1, 2], dtype='int32'))
          out = fluid.layers.less_equal(x=label, y=limit) #out=[True, False]
          out1 = label<= limit #out1=[True, False]

Z
zhoukunsheng 已提交
1491
    """
1492 1493 1494 1495 1496 1497 1498 1499
    check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"],
                             "less_equal")
    check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"],
                             "less_equal")
    if cond is not None:
        check_variable_and_dtype(cond, "cond", [convert_dtype(x.dtype)],
                                 "less_equal")

Z
zhoukunsheng 已提交
1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518
    helper = LayerHelper("less_equal", **locals())
    if cond is None:
        cond = helper.create_variable_for_type_inference(dtype='bool')
        cond.stop_gradient = True

    attrs = dict()

    helper.append_op(
        type='less_equal',
        inputs={'X': [x],
                'Y': [y]},
        outputs={'Out': [cond]},
        attrs=attrs)
    return cond


@templatedoc()
def greater_than(x, y, cond=None):
    """
1519
    This OP returns the truth value of :math:`x > y` elementwise, which is equivalent function to the overloaded operator `>`.
Z
zhoukunsheng 已提交
1520 1521

    Args:
1522 1523 1524 1525 1526
        x(Variable): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. 
        y(Variable): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64.
        cond(Variable, optional): If is :attr:`None`, the op will create a variable as output tensor, the shape and data type of this \
            tensor is the same as input :attr:`x` . If is not :attr:`None`, the op will set the variable as output tensor, the shape and data type \
            of this tensor should be the same as input :attr:`x` . Default value is :attr:`None`.
Z
zhoukunsheng 已提交
1527 1528

    Returns:
1529
        Variable, the output data type is bool.: The tensor variable storing the output, the output shape is the same as input :attr:`x` .
Z
zhoukunsheng 已提交
1530 1531 1532 1533

    Examples:
        .. code-block:: python

1534
          import paddle.fluid as fluid
1535 1536 1537 1538 1539
          import numpy as np
          label = fluid.layers.assign(np.array([2, 3], dtype='int32'))
          limit = fluid.layers.assign(np.array([3, 2], dtype='int32'))
          out = fluid.layers.greater_than(x=label, y=limit) #out=[False, True]
          out1 = label > limit #out1=[False, True]
Z
zhoukunsheng 已提交
1540
    """
1541 1542 1543 1544 1545 1546 1547 1548
    check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"],
                             "greater_than")
    check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"],
                             "greater_than")
    if cond is not None:
        check_variable_and_dtype(cond, "cond", [convert_dtype(x.dtype)],
                                 "greater_than")

Z
zhoukunsheng 已提交
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
    helper = LayerHelper("greater_than", **locals())
    if cond is None:
        cond = helper.create_variable_for_type_inference(dtype='bool')
        cond.stop_gradient = True

    attrs = dict()

    helper.append_op(
        type='greater_than',
        inputs={'X': [x],
                'Y': [y]},
        outputs={'Out': [cond]},
        attrs=attrs)
    return cond


@templatedoc()
def greater_equal(x, y, cond=None):
    """
1568
    This OP returns the truth value of :math:`x >= y` elementwise, which is equivalent function to the overloaded operator `>=`.
Z
zhoukunsheng 已提交
1569 1570

    Args:
1571 1572 1573 1574 1575
        x(Variable): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. 
        y(Variable): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64.
        cond(Variable, optional): If is :attr:`None` , the op will create a variable as output tensor, the shape and data type of this \
            tensor is the same as input :attr:`x`. If is not :attr:`None` , the op will set the variable as output tensor, the shape and data \
            type of this tensor is the same as input :attr:`x`. Default value is :attr:`None`.
Z
zhoukunsheng 已提交
1576 1577

    Returns:
1578
        Variable, the output data type is bool.: The tensor variable storing the output, the output shape is the same as input :attr:`x`.
Z
zhoukunsheng 已提交
1579 1580 1581 1582

    Examples:
        .. code-block:: python

1583
          import paddle.fluid as fluid
1584 1585 1586 1587 1588 1589
          import numpy as np

          label = fluid.layers.assign(np.array([2, 2], dtype='int32'))
          limit = fluid.layers.assign(np.array([2, 3], dtype='int32'))
          out = fluid.layers.greater_equal(x=label, y=limit) #out=[True, False]
          out_1 = label >= limit #out1=[True, False]
1590

Z
zhoukunsheng 已提交
1591
    """
1592 1593 1594 1595 1596 1597 1598 1599
    check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"],
                             "greater_equal")
    check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"],
                             "greater_equal")
    if cond is not None:
        check_variable_and_dtype(cond, "cond", [convert_dtype(x.dtype)],
                                 "greater_equal")

Z
zhoukunsheng 已提交
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
    helper = LayerHelper("greater_equal", **locals())
    if cond is None:
        cond = helper.create_variable_for_type_inference(dtype='bool')
        cond.stop_gradient = True

    attrs = dict()

    helper.append_op(
        type='greater_equal',
        inputs={'X': [x],
                'Y': [y]},
        outputs={'Out': [cond]},
        attrs=attrs)
    return cond


1616
def equal(x, y, cond=None):
1617 1618 1619 1620
    """
    This layer returns the truth value of :math:`x == y` elementwise.

    Args:
W
wangchaochaohu 已提交
1621 1622 1623 1624 1625
        x(Variable): Tensor, data type is float32, float64, int32, int64.
        y(Variable): Tensor, data type is float32, float64, int32, int64.
        cond(Variable, optional): Optional output which can be any created 
            Variable that meets the requirements to store the result of *equal*.
            if cond is None, a new Varibale will be created to store the result.
1626 1627

    Returns:
W
wangchaochaohu 已提交
1628 1629
        Variable: output Tensor, it's shape is the same as the input's Tensor,
        and the data type is bool.
1630 1631 1632 1633

    Examples:
        .. code-block:: python

1634
          import paddle.fluid as fluid
W
wangchaochaohu 已提交
1635 1636 1637 1638 1639 1640 1641
          import numpy as np
          out_cond =fluid.data(name="input1", shape=[2], dtype='bool')
          label = fluid.layers.assign(np.array([3, 3], dtype="int32"))
          limit = fluid.layers.assign(np.array([3, 2], dtype="int32"))
          label_cond = fluid.layers.assign(np.array([1, 2], dtype="int32"))
          out1 = fluid.layers.equal(x=label,y=limit) #out1=[True, False]
          out2 = fluid.layers.equal(x=label_cond,y=limit, cond=out_cond) #out2=[False, True] out_cond=[False, True]
1642
    """
1643 1644 1645 1646 1647 1648 1649 1650
    check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"],
                             "equal")
    check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"],
                             "equal")
    if cond is not None:
        check_variable_and_dtype(cond, "cond", [convert_dtype(x.dtype)],
                                 "equal")

1651 1652
    helper = LayerHelper("equal", **locals())
    if cond is None:
X
Xin Pan 已提交
1653
        cond = helper.create_variable_for_type_inference(dtype='bool')
1654 1655 1656 1657 1658 1659 1660 1661
        cond.stop_gradient = True

    helper.append_op(
        type='equal', inputs={'X': [x],
                              'Y': [y]}, outputs={'Out': [cond]})
    return cond


Z
zhoukunsheng 已提交
1662 1663
def not_equal(x, y, cond=None):
    """
1664
    This OP returns the truth value of :math:`x != y` elementwise, which is equivalent function to the overloaded operator `!=`.
Z
zhoukunsheng 已提交
1665 1666

    Args:
1667 1668 1669 1670 1671
        x(Variable): First input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64. 
        y(Variable): Second input to compare which is N-D tensor. The input data type should be float32, float64, int32, int64.
        cond(Variable, optional): If is :attr:`None`, the op will create a variable as output tensor, the shape and data type of this \
             tensor is the same as input :attr:`x`. If is not :attr:`None`, the op will set the variable as output tensor, the shape and data \
             type of this tensor should be the same as input :attr:`x`. Default value is :attr:`None`.
Z
zhoukunsheng 已提交
1672 1673

    Returns:
1674
        Variable, the output data type is bool.: The tensor variable storing the output, the output shape is the same as input :attr:`x`.
Z
zhoukunsheng 已提交
1675 1676 1677 1678

    Examples:
        .. code-block:: python

1679 1680 1681 1682
          import paddle.fluid as fluid
          
          label = fluid.layers.data(name='label', shape=[1], dtype='int64')
          limit = fluid.layers.fill_constant(shape=[1], value=1, dtype='int64')
Z
zhoukunsheng 已提交
1683 1684
          out = fluid.layers.not_equal(x=label, y=limit)
    """
1685 1686 1687 1688 1689 1690 1691 1692
    check_variable_and_dtype(x, "x", ["float32", "float64", "int32", "int64"],
                             "not_equal")
    check_variable_and_dtype(y, "y", ["float32", "float64", "int32", "int64"],
                             "not_equal")
    if cond is not None:
        check_variable_and_dtype(cond, "cond", [convert_dtype(x.dtype)],
                                 "not_equal")

Z
zhoukunsheng 已提交
1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703
    helper = LayerHelper("not_equal", **locals())
    if cond is None:
        cond = helper.create_variable_for_type_inference(dtype='bool')
        cond.stop_gradient = True

    helper.append_op(
        type='not_equal', inputs={'X': [x],
                                  'Y': [y]}, outputs={'Out': [cond]})
    return cond


1704
def array_read(array, i):
1705
    """
1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720
    This OP is used to read data at the specified position from the input array 
    :ref:`api_fluid_LoDTensorArray` . ``array`` is the input array and ``i``
    is the specified read position. This OP is often used together with 
    :ref:`api_fluid_layers_array_write` OP.

    Case 1:
    ::
        Input:
            The shape of first three tensors are [1], and that of the last one is [1,2]:
                array = ([0.6], [0.1], [0.3], [0.4, 0.2])
            And:
                i = [3]

        Output:
            output = [0.4, 0.2]
1721

K
kavyasrinet 已提交
1722
    Args:
1723 1724 1725
        array (LoDTensorArray): The input LoDTensorArray.
        i (Variable): 1-D Tensor, whose shape is [1] and dtype is int64. It represents the
            specified read position of ``array``.
1726

K
kavyasrinet 已提交
1727
    Returns:
1728
        Variable: The LoDTensor or Tensor that is read at the specified position of ``array``.
1729

K
kavyasrinet 已提交
1730
    Examples:
1731 1732
        .. code-block:: python

1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763
            # First we're going to create a LoDTensorArray, then we're going to write the Tensor into
            # the specified position, and finally we're going to read the Tensor at that position.
            import paddle.fluid as fluid
            arr = fluid.layers.create_array(dtype='float32')
            tmp = fluid.layers.fill_constant(shape=[3, 2], dtype='int64', value=5)
            i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10)
            # tmp is the Tensor with shape [3,2], and if we write it into the position with subscript 10
            # of the empty-array: arr, then the length of arr becomes 11.
            arr = fluid.layers.array_write(tmp, i, array=arr)
            # Read the data of the position with subscript 10.
            item = fluid.layers.array_read(arr, i)

            # You can print out the data via executor.
            input = fluid.layers.Print(item, message="The LoDTensor of the i-th position:")
            main_program = fluid.default_main_program()
            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(main_program)

            # The printed result is:

            # 1569588169  The LoDTensor of the i-th position: The place is:CPUPlace
            # Tensor[array_read_0.tmp_0]
            #    shape: [3,2,]
            #    dtype: l
            #    data: 5,5,5,5,5,5,

            # the output is 2-D Tensor with shape [3,2].
            # dtype is the corresponding C++ data type, which may vary in different environments.
            # Eg: if the data type of tensor is int64, then the corresponding C++ data type is int64_t, 
            #       so the dtype value is typeid(int64_t).Name(), which is 'x' on MacOS, 'l' on Linux, 
            #       and '__int64' on Windows. They both represent 64-bit integer variables.
1764
    """
1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
    if in_dygraph_mode():
        assert isinstance(
            array,
            list), "The 'array' in array_read must be list in dygraph mode"
        assert isinstance(
            i, Variable
        ), "The index 'i' in array_read must be Variable in dygraph mode"
        assert i.shape == [
            1
        ], "The shape of index 'i' should be [1] in dygraph mode"
        i = i.numpy()[0]
        return array[i]

1778
    check_variable_and_dtype(i, 'i', ['int64'], 'array_read')
Y
Yu Yang 已提交
1779 1780 1781 1782 1783
    helper = LayerHelper('array_read', **locals())
    if not isinstance(
            array,
            Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY:
        raise TypeError("array should be tensor array vairable")
X
Xin Pan 已提交
1784
    out = helper.create_variable_for_type_inference(dtype=array.dtype)
Y
Yu Yang 已提交
1785 1786 1787 1788 1789 1790
    helper.append_op(
        type='read_from_array',
        inputs={'X': [array],
                'I': [i]},
        outputs={'Out': [out]})
    return out
Y
Yang Yu 已提交
1791 1792


1793
def shrink_memory(x, i, table):
1794
    """
Y
yuyang18 已提交
1795
    This function creates an operator to shrink rnn memory using the RankTable
1796
    as mentioned in the input parameter.
Y
yuyang18 已提交
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816

    NOTE: This API is very low-level API. It is used by DynamicRNN only.

    Since the Dynamic RNN uses no-padding way to implement RNN. The sequence
    will be sorted by order, and the length of valid memory will be shrink after
    each time step.

    Args:
        x(Variable): The memory object in the previous time step.
        i(Variable): The step count variable. A int scalar as LoDTensor.
        table(Variable): The RNNRankTable object.

    Returns:
        the memory variable after shrink.

    Examples:

        Since this API is very low level API. The example is not provided.
        Please reference the implementation of class DynamicRNN for detail
        usage.
1817
    """
Y
Yang Yu 已提交
1818
    helper = LayerHelper('shrink_memory', **locals())
X
Xin Pan 已提交
1819
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
Y
Yang Yu 已提交
1820
    helper.append_op(
Y
Yang Yu 已提交
1821
        type='shrink_rnn_memory',
Y
Yang Yu 已提交
1822 1823 1824 1825 1826 1827
        inputs={'X': [x],
                'I': [i],
                'RankTable': [table]},
        outputs={'Out': [out]},
        attrs={})
    return out
Y
Yang Yu 已提交
1828 1829


1830
def array_length(array):
1831
    """
1832 1833
    This OP is used to get the length of the input array :ref:`api_fluid_LoDTensorArray` .
    It can be used together with :ref:`api_fluid_layers_array_read` , :ref:`api_fluid_layers_array_write` , 
T
tianshuo78520a 已提交
1834
    :ref:`api_fluid_layers_While` OP to traverse, read and write LoDTensorArray.
1835

K
kavyasrinet 已提交
1836
    Args:
1837
        array (LoDTensorArray): The input array that will be used to compute the length.
K
kavyasrinet 已提交
1838 1839

    Returns:
1840
        Variable: 1-D Tensor with shape [1], which is the length of array. Datatype: int64.
K
kavyasrinet 已提交
1841 1842

    Examples:
Q
qiaolongfei 已提交
1843
        .. code-block:: python
K
kavyasrinet 已提交
1844

1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860
            import paddle.fluid as fluid
            tmp = fluid.layers.zeros(shape=[10], dtype='int32')
            i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=10)
            # tmp is 1-D Tensor with shape [10]. We write tmp into arr on subscript 10,
            # then the length of arr becomes 11.
            arr = fluid.layers.array_write(tmp, i=i)
            # return the length of arr
            arr_len = fluid.layers.array_length(arr)

            # You can use executor to print out the length of LoDTensorArray.
            input = fluid.layers.Print(arr_len, message="The length of LoDTensorArray:")
            main_program = fluid.default_main_program()
            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(main_program)

            # The printed result is:
Q
qiaolongfei 已提交
1861

1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873
            # 1569576542  The length of LoDTensorArray:   The place is:CPUPlace
            # Tensor[array_length_0.tmp_0]
            #    shape: [1,]
            #    dtype: l
            #    data: 11,
            
            # 1-D Tensor with shape [1], whose value is 11. It means that the length of LoDTensorArray
            # is 11.
            # dtype is the corresponding C++ data type, which may vary in different environments.
            # Eg: if the data type of tensor is int64, then the corresponding C++ data type is int64_t, 
            #       so the dtype value is typeid(int64_t).Name(), which is 'x' on MacOS, 'l' on Linux, 
            #       and '__int64' on Windows. They both represent 64-bit integer variables.
1874
    """
1875

1876 1877 1878 1879 1880 1881
    if in_dygraph_mode():
        assert isinstance(
            array,
            list), "The 'array' in array_write must be a list in dygraph mode"
        return len(array)

1882 1883 1884 1885 1886 1887
    if not isinstance(
            array,
            Variable) or array.type != core.VarDesc.VarType.LOD_TENSOR_ARRAY:
        raise TypeError(
            "array should be tensor array vairable in array_length Op")

Y
Yang Yu 已提交
1888
    helper = LayerHelper('array_length', **locals())
X
Xin Pan 已提交
1889
    tmp = helper.create_variable_for_type_inference(dtype='int64')
Y
Yang Yu 已提交
1890 1891 1892 1893
    tmp.stop_gradient = True
    helper.append_op(
        type='lod_array_length', inputs={'X': [array]}, outputs={'Out': [tmp]})
    return tmp
Y
Yu Yang 已提交
1894 1895 1896


class ConditionalBlockGuard(BlockGuard):
F
fengjiayi 已提交
1897
    """
1898 1899 1900
    ConditionalBlockGuard is derived from BlockGuard. It is dedicated for
    holding a ConditionalBlock, and helping users entering and exiting the
    ConditionalBlock via Python's 'with' keyword. However, ConditionalBlockGuard
F
fengjiayi 已提交
1901 1902 1903
    is generally an internal component of IfElse, users should not use it directly.
    """

Y
Yu Yang 已提交
1904
    def __init__(self, block):
1905
        check_type(block, "block", ConditionalBlock, "ConditionalBlockGuard")
Y
Yu Yang 已提交
1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
        super(ConditionalBlockGuard, self).__init__(block.helper.main_program)
        self.block = block

    def __enter__(self):
        return super(ConditionalBlockGuard, self).__enter__()

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.block.complete()
        return super(ConditionalBlockGuard, self).__exit__(exc_type, exc_val,
                                                           exc_tb)


class ConditionalBlock(object):
Y
Yan Chunwei 已提交
1919 1920 1921 1922 1923 1924 1925 1926
    '''
    **ConditionalBlock**

    ConditionalBlock is an operator that bind a block to a specific condition,
    if the condition matches, the corresponding block will be executed.

    Args:
        inputs (Variable): bool conditions.
T
tianshuo78520a 已提交
1927
        is_scalar_condition (bool): whether the branch is controlled by a scalar.
Y
Yan Chunwei 已提交
1928 1929 1930 1931 1932
        name(str): name of this ConditionalBlock.

    Examples:
        .. code-block:: python

1933
             import paddle.fluid as fluid
Y
Yan Chunwei 已提交
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944
             cond = layers.less_than(x=label, y=limit)
             true_image, false_image = layers.split_lod_tensor(
                 input=image, mask=cond)
             true_cond = layers.ConditionalBlock([true_image])

             with true_cond.block():
                 ...
             with false_cond.block():
                 ...
    '''

1945
    def __init__(self, inputs, is_scalar_condition=False, name=None):
Y
Yu Yang 已提交
1946
        for each_input in inputs:
1947
            check_type(each_input, "input", Variable, "ConditionalBlock")
Y
Yu Yang 已提交
1948
        self.inputs = inputs
1949
        self.is_scalar_condition = is_scalar_condition
1950
        self.helper = LayerHelper('conditional_block', name=name)
Y
Yu Yang 已提交
1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973

    def block(self):
        return ConditionalBlockGuard(self)

    def complete(self):
        inside_block = self.helper.main_program.current_block()
        parent_block = self.helper.main_program.block(inside_block.parent_idx)

        intermediate = set()
        params = set()

        for each_op in inside_block.ops:
            assert isinstance(each_op, Operator)
            for iname in each_op.input_names:
                for in_var_name in each_op.input(iname):
                    if in_var_name not in intermediate:
                        params.add(in_var_name)

            for oname in each_op.output_names:
                for out_var_name in each_op.output(oname):
                    intermediate.add(out_var_name)
        input_set = set([ipt.name for ipt in self.inputs])

1974 1975 1976
        # Todo(liym27) Here assume that all params are in recursive parent block
        # but when minimize() called in control flow, some params may be in
        # conditional grad block
Y
Yu Yang 已提交
1977
        param_list = [
W
Wu Yi 已提交
1978
            parent_block._var_recursive(each_name) for each_name in params
Y
Yu Yang 已提交
1979 1980
        ]

X
Xin Pan 已提交
1981 1982 1983 1984 1985
        out_list = []
        for inner_out_name in intermediate:
            inner_var = parent_block._find_var_recursive(inner_out_name)
            if inner_var:
                out_list.append(inner_var)
Y
Yu Yang 已提交
1986 1987

        step_scope = parent_block.create_var(
1988
            type=core.VarDesc.VarType.STEP_SCOPES)
1989
        conditional_block_op = parent_block.append_op(
Y
Yu Yang 已提交
1990 1991
            type='conditional_block',
            inputs={
1992 1993
                'Cond': self.inputs,
                'Input': param_list,
Y
Yu Yang 已提交
1994 1995 1996
            },
            outputs={'Out': out_list,
                     'Scope': [step_scope]},
1997 1998 1999 2000 2001
            attrs={
                'sub_block': inside_block,
                'is_scalar_condition': self.is_scalar_condition
            })

2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
        if self.need_append_conditional_block_grad(inside_block):
            self.append_conditional_block_grad(parent_block, inside_block,
                                               conditional_block_op)

    def need_append_conditional_block_grad(self, inside_block):
        grad_sub_block_idx = inside_block.backward_block_idx

        return grad_sub_block_idx != -1

    def append_conditional_block_grad(self, parent_block, inside_block,
                                      conditional_block_op):
        '''
        Append op `conditional_block_grad` manually.
        When `optimizer.minimize/append_backward` is called in Paddle control flow,
        grad ops will be appended before appending op `conditional_block` so that
        op `conditional_block_grad` can't be appended when calling
        `optimizer.minimize/append_backward`. After appending op `conditional_block`,
        `conditional_block_grad` is appended manually.

        Args:
            parent_block (Block): The block that `conditional_block_op` blongs to.
            inside_block (Block): The sub block of `conditional_block_op`.
            conditional_block_op (Operator): The forward op conditional_block.
        '''

        grad_sub_block_idx = inside_block.backward_block_idx
        grad_sub_block = self.helper.main_program.block(grad_sub_block_idx)

        intermediate = set()
        params = set()

        for each_op in grad_sub_block.ops:
            assert isinstance(each_op, Operator)
            for iname in each_op.input_names:
                for in_var_name in each_op.input(iname):
                    if in_var_name not in intermediate:
                        params.add(in_var_name)

            for oname in each_op.output_names:
                for out_var_name in each_op.output(oname):
                    intermediate.add(out_var_name)

        param_list = []
        for inner_input_name in params:
            inner_var = parent_block._find_var_recursive(inner_input_name)
            if inner_var:
                param_list.append(cpt.to_text(inner_var.name))

        grad_op_desc, op_grad_to_var = core.get_grad_op_desc(
            conditional_block_op.desc,
            cpt.to_text(set()), [grad_sub_block.desc])

        # append op_desc in grad_op_descs to target_block
        op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName()
        backward = core.op_proto_and_checker_maker.OpRole.Backward
        new_op_desc = parent_block.desc.append_op()
        new_op_desc.copy_from(grad_op_desc[0])
        new_op_desc._set_attr(op_role_attr_name, backward)
        # set input and output manually
        new_op_desc.set_input('Input', param_list)
        new_op_desc.set_output('Input@GRAD',
                               [param + "@GRAD" for param in param_list])

        new_vars = set()
        for grad_var_name in new_op_desc.output_arg_names():
            if grad_sub_block.desc.has_var_recursive(
                    cpt.to_bytes(grad_var_name)
            ) or grad_var_name == core.empty_var_name():
                continue
            grad_sub_block.desc.var(cpt.to_bytes(grad_var_name))
            new_vars.add(grad_var_name)
            if grad_var_name not in op_grad_to_var:
                continue

        # infer_shape and infer_type
        new_op_desc.infer_var_type(grad_sub_block.desc)
        new_op_desc.infer_shape(grad_sub_block.desc)

        for arg in new_op_desc.output_arg_names():
            if arg in new_vars:
                _infer_var_data_type_shape_(arg, grad_sub_block)

        self.helper.main_program._sync_with_cpp()

2086

2087 2088 2089 2090 2091 2092 2093 2094
def copy_var_to_parent_block(var, layer_helper):
    if var is None:
        return None
    prog = layer_helper.main_program
    parent_idx = prog.current_block().parent_idx
    assert parent_idx >= 0, "Got wrong parent block index when assigning var to parent scope in control_flow"
    parent_block = prog.block(parent_idx)

2095 2096
    parent_block_var = parent_block.create_var(
        dtype=var.dtype, shape=var.shape, type=var.type)
2097 2098 2099 2100 2101 2102
    assign(var, parent_block_var)
    return parent_block_var


def cond(pred, true_fn=None, false_fn=None, name=None):
    """
2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113
    This API returns ``true_fn()`` if the predicate ``pred`` is true else
    ``false_fn()`` . Users could also set ``true_fn`` or ``false_fn`` to
    ``None`` if do nothing and this API will treat the callable simply returns
    ``None`` in this case.

    ``true_fn`` and ``false_fn`` should return same nest structure of tensors
    or both return ``None`` if user doens't like to return anything. A nest
    structure of tensors in PaddlePaddle is tensor(s), or tuple of tensors, or
    list of tensors.
    
    Note: 
2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131
        1. The tuples or lists returned by ``true_fn`` and ``false_fn`` must have
        the same shape because of dataflow model of PaddlePaddle while the
        tensors in the tuples or the lists can have different shapes.

        2. Any tensors or operations created outside of ``true_fn`` and
        ``false_fn`` will be executed regardless of which branch is selected at
        runtime. This has frequently surprised users who expected a lazy
        semantics. For example:

        .. code-block:: python
        
            import paddle.fluid as fluid
            a = fluid.data(name='a', shape=[-1, 1], dtype='float32')
            b = fluid.data(name='b', shape=[-1, 1], dtype='float32')
            c = a * b
            out = fluid.layers.cond(a < b, lambda: a + c, lambda: b * b)

        No matter whether ``a < b`` , ``c = a * b`` will run.
2132 2133 2134 2135

    Args:
        pred(Variable): A boolean tensor whose numel should be 1. The boolean
            value determines whether to return the result of ``true_fn`` or
2136 2137 2138 2139 2140 2141
            ``false_fn`` .
        true_fn(callable, optional): A callable to be performed if ``pred`` is
            true. The default value is ``None`` .
        false_fn(callable, optional): A callable to be performed if ``pred`` is
            false. The default value is ``None`` .
        name(str, optional): The default value is ``None`` . Normally users
2142
             don't have to set this parameter. For more information, please
2143 2144 2145 2146 2147
             refer to :ref:`api_guide_Name` .

    Returns:
        Variable|list(Variable)|tuple(Variable): returns ``true_fn()`` if the
        predicate ``pred`` is true else ``false_fn()`` .
2148 2149 2150

    Raises:
        TypeError: if ``true_fn`` or ``false_fn`` is not callable.
2151 2152
        ValueError: if ``true_fn`` and ``false_fn`` don't return the same nest
            structure of tensors.
2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
            import paddle.fluid.layers as layers
            from paddle.fluid.executor import Executor
            from paddle.fluid.framework import Program, program_guard

            #
            # pseudocode:
            # if 0.1 < 0.23:
            #     return 1, True
            # else:
            #     return 3, 2
            #

            def true_func():
                return layers.fill_constant(
                    shape=[1, 2], dtype='int32', value=1), layers.fill_constant(
                        shape=[2, 3], dtype='bool', value=True)

            def false_func():
                return layers.fill_constant(
                    shape=[3, 4], dtype='float32', value=3), layers.fill_constant(
                        shape=[4, 5], dtype='int64', value=2)

            main_program = Program()
            startup_program = Program()
            with program_guard(main_program, startup_program):
                x = layers.fill_constant(shape=[1], dtype='float32', value=0.1)
                y = layers.fill_constant(shape=[1], dtype='float32', value=0.23)
                pred = layers.less_than(x, y)            
                out = layers.cond(pred, true_func, false_func)
                # out is a tuple containing 2 tensors

            place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda(
            ) else fluid.CPUPlace()
            exe = fluid.Executor(place)
            ret = exe.run(main_program, fetch_list=out)
            # ret[0] = [[1 1]]
            # ret[1] = [[ True  True  True]
            #           [ True  True  True]]

2197
    """
2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217
    if in_dygraph_mode():
        assert isinstance(pred, Variable), "The pred in cond must be Variable"
        assert pred.numpy().size == 1, "condition input's numel should be 1"
        pred = pred.numpy()[0]
        if pred:
            if true_fn is not None:
                if not callable(true_fn):
                    raise TypeError(
                        "The true_fn in cond must be callable, but received {}".
                        format(type(true_fn).__name__))
                return true_fn()
        else:
            if false_fn is not None:
                if not callable(false_fn):
                    raise TypeError(
                        "The false_fn in cond must be callable, but received {}".
                        format(type(false_fn).__name__))
                return false_fn()
        return None

2218 2219
    check_variable_and_dtype(pred, "pred", ['bool'], "fluid.layers.cond")
    check_type(name, "name", (str, type(None)), "fluid.layers.cond")
2220 2221 2222
    helper = LayerHelper('cond', **locals())
    true_output = None
    false_output = None
2223
    copy_to_parent_func = lambda var: copy_var_to_parent_block(var, helper)
2224 2225
    if true_fn is not None:
        if not callable(true_fn):
2226 2227 2228
            raise TypeError(
                "The true_fn in cond must be callable, but received {}".format(
                    type(true_fn).__name__))
2229 2230 2231 2232
        true_cond_block = ConditionalBlock([pred], is_scalar_condition=True)
        with true_cond_block.block():
            origin_true_output = true_fn()
            if origin_true_output is not None:
2233
                true_output = map_structure(copy_to_parent_func,
2234 2235 2236
                                            origin_true_output)
    if false_fn is not None:
        if not callable(false_fn):
2237 2238 2239
            raise TypeError(
                "The false_fn in cond must be callable, but received {}".format(
                    type(false_fn).__name__))
2240 2241 2242 2243 2244
        false_cond_block = ConditionalBlock(
            [logical_not(pred)], is_scalar_condition=True)
        with false_cond_block.block():
            origin_false_output = false_fn()
            if origin_false_output is not None:
2245
                false_output = map_structure(copy_to_parent_func,
2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
                                             origin_false_output)

    if true_output is None and false_output is None:
        return None

    if true_output is None:
        raise ValueError(
            "Incompatible return values of true_fn and false_fn in cond: "
            "true_fn returns None while false_fn returns non-None")
    if false_output is None:
        raise ValueError(
            "Incompatible return values of true_fn and false_fn in cond: "
            "true_fn returns non-None while false_fn returns None")

    # Merge ture and false output if they are not None
    try:
        assert_same_structure(true_output, false_output, check_types=False)
    except ValueError as e:
        raise ValueError(
            "Incompatible return values of true_fn and false_fn in cond: {}".
            format(e))

    mask = cast(pred, dtype='int32')
    merge_func = lambda false_var, true_var : select_input([false_var, true_var], mask)
    merged_output = map_structure(merge_func, false_output, true_output)
    return merged_output


L
liym27 已提交
2274
def _error_message(what, arg_name, op_name, right_value, error_value):
2275
    error_message = "{what} of '{arg_name}' in {op_name} must be " \
L
liym27 已提交
2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311
        "{right_value}, but received: {error_value}.".format(
        what=what,
        arg_name=arg_name,
        op_name=op_name,
        right_value=right_value,
        error_value=error_value)

    return error_message


def case(pred_fn_pairs, default=None, name=None):
    '''
    This operator works like an if-elif-elif-else chain.

    Args:
        pred_fn_pairs(list|tuple): A list or tuple of (pred, fn) pairs. ``pred`` is a boolean Tensor with shape [1], ``fn`` is a callable. All callables return the same structure of Tensors.
        default(callable, optional): Callable that returns a structure of Tensors.
        name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`.

    Returns:
        Variable|list(Variable): Tensors returned by the callable from the first pair whose pred is True,
        or Tensors returned by ``default`` if no pred in ``pred_fn_pairs`` is True and ``default`` is not None,
        or Tensors returned by the last callable in ``pred_fn_pairs``  if no pred in ``pred_fn_pairs`` is True and ``default`` is None.

    Raises:
        TypeError: If the type of ``pred_fn_pairs`` is not list or tuple.
        TypeError: If the type of elements in ``pred_fn_pairs`` is not tuple.
        TypeError: If the size of tuples in ``pred_fn_pairs`` is not 2.
        TypeError: If the first element of 2-tuple in ``pred_fn_pairs`` is not Variable.
        TypeError: If the second element of 2-tuple in ``pred_fn_pairs`` is not callable.
        TypeError: If ``default`` is not None but it is not callable.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
2312
            import paddle.fluid.layers as layers
L
liym27 已提交
2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324

            def fn_1():
                return layers.fill_constant(shape=[1, 2], dtype='float32', value=1)

            def fn_2():
                return layers.fill_constant(shape=[2, 2], dtype='int32', value=2)

            def fn_3():
                return layers.fill_constant(shape=[3], dtype='int32', value=3)

            main_program = fluid.default_startup_program()
            startup_program = fluid.default_main_program()
2325
            with fluid.program_guard(main_program, startup_program):
L
liym27 已提交
2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352
                x = layers.fill_constant(shape=[1], dtype='float32', value=0.3)
                y = layers.fill_constant(shape=[1], dtype='float32', value=0.1)
                z = layers.fill_constant(shape=[1], dtype='float32', value=0.2)

                pred_1 = layers.less_than(z, x)  # true: 0.2 < 0.3
                pred_2 = layers.less_than(x, y)  # false: 0.3 < 0.1
                pred_3 = layers.equal(x, y)      # false: 0.3 == 0.1

                # Call fn_1 because pred_1 is True
                out_1 = layers.case(
                    pred_fn_pairs=[(pred_1, fn_1), (pred_2, fn_2)], default=fn_3)

                # Argument default is None and no pred in pred_fn_pairs is True. fn_3 will be called.
                # because fn_3 is the last callable in pred_fn_pairs.
                out_2 = layers.case(pred_fn_pairs=[(pred_2, fn_2), (pred_3, fn_3)])

                exe = fluid.Executor(fluid.CPUPlace())
                res_1, res_2 = exe.run(main_program, fetch_list=[out_1, out_2])
                print(res_1)  # [[1. 1.]]
                print(res_2)  # [3 3 3]
    '''
    helper = LayerHelper('case', **locals())

    def _case_check_args(pred_fn_pairs, default):
        '''
        Check arguments pred_fn_pairs and default. Return canonical pre_fn_pairs and default.
        '''
2353
        check_type(pred_fn_pairs, 'pred_fn_pairs', (list, tuple), 'case')
L
liym27 已提交
2354 2355 2356 2357 2358

        for pred_fn in pred_fn_pairs:
            if not isinstance(pred_fn, tuple):
                raise TypeError(
                    _error_message("The elements' type", "pred_fn_pairs",
2359
                                   "case", tuple, type(pred_fn)))
L
liym27 已提交
2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395
            if len(pred_fn) != 2:
                raise TypeError(
                    _error_message("The tuple's size", "pred_fn_pairs", "case",
                                   "2", str(len(pred_fn)) + "-tuple"))
            pred, fn = pred_fn

            if not isinstance(pred, Variable):
                raise TypeError(
                    _error_message("The pred's type", "pred_fn_pairs", "case",
                                   "boolean Variable", type(pred)))

            if not callable(fn):
                raise TypeError(
                    "The fn for {} of pred_fn_pairs in Op(case) must"
                    " be callable.".format(pred.name))

        if default is None:
            default_index = len(pred_fn_pairs) - 1  # pick the last one
            default = pred_fn_pairs[default_index][1]
            pred_fn_pairs = pred_fn_pairs[:default_index]
        elif not callable(default):
            raise TypeError("The default in Op(case) must be callable.")

        return pred_fn_pairs, default

    pred_fn_pairs, default = _case_check_args(pred_fn_pairs, default)

    false_fn = default
    for pred, true_fn in reversed(pred_fn_pairs):
        false_fn = partial(cond, pred=pred, true_fn=true_fn, false_fn=false_fn)

    final_fn = false_fn

    return final_fn()


2396
class Switch(object):
Q
qiaolongfei 已提交
2397 2398
    """

2399 2400 2401 2402 2403 2404 2405
    This class is used to implement Switch branch control function. 
    Switch branch contains several case branches and one default branch. 
    Switch control flow checks whether the case branch conditions are satisfied in turn, 
    and only executes the statement after the first case branch that satisfies the conditions. 
    If there is no case branch that satisfies the condition, 
    only the statement following the default branch is executed.

2406 2407 2408 2409
    Note:
        A new OP :ref:`api_fluid_layers_case` is highly recommended instead of ``Switch`` if the shape of parameter ``cond`` is [1].
        OP :ref:`api_fluid_layers_case` is easier to use and is called with less code but does the same thing as ``Switch`` .

2410
    Member Functions:
2411
        case(condition): The case branch of Switch whose parameter cond is a scalar Variable of bool type. Only if the cond of the current case branch is True and the cond of the previous case branch is False, the statement after the case branch will be executed, and the statement after the case branch will not be executed.
2412 2413 2414 2415 2416 2417
        
        default(): The default branch of Switch. When cond of all case branches is False, the statement after default branch is executed.

    Case and default functions can only be used inside the scope of Switch, as shown below:

    .. code-block:: python
2418

2419 2420 2421 2422 2423 2424 2425 2426 2427
        '''
        with fluid.layers.Switch() as switch:
            with switch.case(cond1):
                i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=1)
            with switch.case(cond2):
                i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=2)
            with switch.default():
                i = fluid.layers.fill_constant(shape=[1], dtype='int64', value=0)
        '''
Q
qiaolongfei 已提交
2428

2429 2430
    Args:
        name(str, optional): The default value is None.  Normally there is no need for user to set this property.  For more information, please refer to :ref:`api_guide_Name` .
Q
qiaolongfei 已提交
2431 2432 2433

    Examples:
        .. code-block:: python
2434 2435
            
            import paddle.fluid as fluid
Q
qiaolongfei 已提交
2436

2437
            lr = fluid.layers.create_global_var(
Q
qiaolongfei 已提交
2438 2439 2440 2441 2442
                shape=[1],
                value=0.0,
                dtype='float32',
                persistable=True,
                name="learning_rate")
2443
            zero_var = fluid.layers.fill_constant(
2444
                shape=[1], dtype='float32', value=0.0)
2445
            one_var = fluid.layers.fill_constant(
Q
qiaolongfei 已提交
2446
                shape=[1], dtype='float32', value=1.0)
2447
            two_var = fluid.layers.fill_constant(
2448
                shape=[1], dtype='float32', value=2.0)
2449

2450
            global_step = fluid.layers.autoincreased_step_counter(counter_name='@LR_DECAY_COUNTER@', begin=0, step=1)
Q
qiaolongfei 已提交
2451 2452

            with fluid.layers.control_flow.Switch() as switch:
Q
qiaolongfei 已提交
2453
                with switch.case(global_step == zero_var):
2454
                    fluid.layers.assign(input=one_var, output=lr)
Q
qiaolongfei 已提交
2455
                with switch.default():
2456
                    fluid.layers.assign(input=two_var, output=lr)
Q
qiaolongfei 已提交
2457

2458 2459 2460 2461 2462
            exe = fluid.Executor(fluid.CPUPlace())
            exe.run(fluid.default_startup_program())

            res = exe.run(fluid.default_main_program(), feed={}, fetch_list=[lr])
            print(res) # [array([1.], dtype=float32)]
Q
qiaolongfei 已提交
2463 2464
    """

2465 2466 2467 2468 2469 2470 2471 2472 2473
    def __init__(self, name=None):
        self.helper = LayerHelper('switch', name=name)
        self.inside_scope = False
        self.pre_not_conditions = []

    def case(self, condition):
        if not self.inside_scope:
            raise ValueError("case should be called inside with")

2474 2475 2476 2477
        check_variable_and_dtype(
            condition, 'condition', ['bool'],
            'the member function case of fluid.layers.Switch')

2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517
        if len(self.pre_not_conditions) == 0:
            cond_block = ConditionalBlock([condition], is_scalar_condition=True)
            not_cond = logical_not(x=condition)
            self.pre_not_conditions.append(not_cond)
        else:
            pre_cond_num = len(self.pre_not_conditions)
            pre_not_cond = self.pre_not_conditions[pre_cond_num - 1]
            new_not_cond = logical_and(
                x=pre_not_cond, y=logical_not(x=condition))
            self.pre_not_conditions.append(new_not_cond)
            cond_block = ConditionalBlock(
                [logical_and(
                    x=pre_not_cond, y=condition)],
                is_scalar_condition=True)

        return ConditionalBlockGuard(cond_block)

    def default(self):
        pre_cond_num = len(self.pre_not_conditions)
        if pre_cond_num == 0:
            raise ValueError("there should be at least one condition")
        cond_block = ConditionalBlock(
            [self.pre_not_conditions[pre_cond_num - 1]],
            is_scalar_condition=True)
        return ConditionalBlockGuard(cond_block)

    def __enter__(self):
        """
        set flag that now is inside switch.block {}
        :return:
        """
        self.inside_scope = True
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.inside_scope = False
        if exc_type is not None:
            return False  # re-raise exception

        return True
Y
Yu Yang 已提交
2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553


class IfElseBlockGuard(object):
    def __init__(self, is_true, ifelse):
        if not isinstance(ifelse, IfElse):
            raise TypeError("ifelse must be an instance of IfElse class")

        if ifelse.status != IfElse.OUT_IF_ELSE_BLOCKS:
            raise ValueError("You cannot invoke IfElse.block() inside a block")

        self.is_true = is_true
        self.ie = ifelse
        if is_true:
            self.cond_block = ifelse.conditional_true_block
        else:
            self.cond_block = ifelse.conditional_false_block

        if not isinstance(self.cond_block, ConditionalBlock):
            raise TypeError("Unexpected situation")

        self.cond_block = self.cond_block.block()

    def __enter__(self):
        self.ie.status = IfElse.IN_IF_ELSE_TRUE_BLOCKS if self.is_true else IfElse.IN_IF_ELSE_FALSE_BLOCKS
        self.cond_block.__enter__()

    def __exit__(self, exc_type, exc_val, exc_tb):
        if not self.cond_block.__exit__(exc_type, exc_val, exc_tb):
            # re-raise inside exception
            return False
        if len(self.ie.output_table[1 if self.is_true else 0]) == 0:
            raise ValueError("Must set output inside block")
        self.ie.status = IfElse.OUT_IF_ELSE_BLOCKS


class IfElse(object):
X
Xin Pan 已提交
2554
    """
2555 2556 2557 2558
    This class is used to implement IfElse branch control function. IfElse contains two blocks, true_block and false_block. IfElse will put data satisfying True or False conditions into different blocks to run.

    Cond is a 2-D Tensor with shape [N, 1] and data type bool, representing the execution conditions of the corresponding part of the input data.

2559 2560 2561 2562
    Note:
        A new OP :ref:`api_fluid_layers_cond` is highly recommended instead of ``IfElse``. if the shape of parameter ``cond`` is [1].
        OP :ref:`api_fluid_layers_cond` is easier to use and is called with less code but does the same thing as ``IfElse`` .

2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603
    IfElse OP is different from other OPs in usage, which may cause some users confusion. Here is a simple example to illustrate this OP.

    .. code-block:: python
        
        # The following code completes the function: subtract 10 from the data greater than 0 in x, add 10 to the data less than 0 in x, and sum all the data.
        import numpy as np
        import paddle.fluid as fluid

        x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32', append_batch_size=False)
        y = fluid.layers.data(name='y', shape=[4, 1], dtype='float32', append_batch_size=False)

        x_d = np.array([[3], [1], [-2], [-3]]).astype(np.float32)
        y_d = np.zeros((4, 1)).astype(np.float32)
        
        # Compare the size of x, y pairs of elements, output cond, cond is shape [4, 1], data type bool 2-D tensor.
        # Based on the input data x_d, y_d, it can be inferred that the data in cond are [[true], [true], [false], [false]].
        cond = fluid.layers.greater_than(x, y)
        # Unlike other common OPs, ie below returned by the OP is an IfElse OP object
        ie = fluid.layers.IfElse(cond)

        with ie.true_block():
            # In this block, according to cond condition, the data corresponding to true dimension in X is obtained and subtracted by 10.
            out_1 = ie.input(x)
            out_1 = out_1 - 10
            ie.output(out_1)
        with ie.false_block():
            # In this block, according to cond condition, get the data of the corresponding condition in X as false dimension, and add 10
            out_1 = ie.input(x)
            out_1 = out_1 + 10
            ie.output(out_1)

        # According to cond condition, the data processed in the two blocks are merged. The output here is output, the type is List, and the element type in List is Variable.
        output = ie() #  [array([[-7.], [-9.], [ 8.], [ 7.]], dtype=float32)] 

        # Get the first Variable in the output List and add all elements.
        out = fluid.layers.reduce_sum(output[0])

        exe = fluid.Executor(fluid.CPUPlace())
        exe.run(fluid.default_startup_program())

        res = exe.run(fluid.default_main_program(), feed={"x":x_d, "y":y_d}, fetch_list=[out])
2604
        print(res)
2605
        # [array([-1.], dtype=float32)] 
X
Xin Pan 已提交
2606 2607

    Args:
2608 2609
        cond (Variable): cond is a 2-D Tensor with shape [N, 1] and data type bool, representing the corresponding execution conditions of N input data. The data type is bool.
        name(str, optional): The default value is None.  Normally there is no need for user to set this property.  For more information, please refer to :ref:`api_guide_Name` .
X
Xin Pan 已提交
2610

2611 2612
    Returns:
        Unlike other common OPs, the OP call returns an IfElse OP object (e.g. ie in the example), which branches the input data by calling the internal functions of the object ``true_block ()``, ``false_block ()``, ``input ()``, ``output ()``, and integrates the data processed by different branches as the overall output by calling the internal ``call ()`` function. The output type is a list, and the type of each element in the list is Variable.
X
Xin Pan 已提交
2613

2614 2615 2616 2617 2618 2619 2620 2621 2622 2623
    Internal Functions:
        The block is constructed by calling the ``with ie. true_block()`` function in the object, and the computational logic under condition true is put into the block. If no corresponding block is constructed, the input data in the corresponding conditional dimension is unchanged.
 
        The block is constructed by calling the ``with ie. false_block()`` function in the object, and the computational logic under condition false is put into the block. If no corresponding block is constructed, the input data in the corresponding conditional dimension is unchanged.

        ``Out = ie. input (x)`` will take out the data of the corresponding conditional dimension in X and put it into out, supporting the internal processing of multiple inputs in block.

        ``ie. output (out)`` writes the result to the output of the corresponding condition.

        There is a ``call ()`` function inside the object, that is, by calling ``output = ie ()``, all the outputs inside the block of False are fused as the whole output, the output type is a list, and the type of each element in the list is Variable.
2624

X
Xin Pan 已提交
2625
    """
Y
Yu Yang 已提交
2626 2627 2628 2629
    OUT_IF_ELSE_BLOCKS = 0
    IN_IF_ELSE_TRUE_BLOCKS = 1
    IN_IF_ELSE_FALSE_BLOCKS = 2

2630
    def __init__(self, cond, name=None):
2631 2632
        check_type(cond, "cond", Variable, "fluid.layers.IfElse")
        check_type(name, "name", (str, type(None)), "fluid.layers.IfElse")
2633
        self.helper = LayerHelper('ifelse', name=name)
Y
Yu Yang 已提交
2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644
        self.cond = cond
        self.input_table = {}
        self.status = IfElse.OUT_IF_ELSE_BLOCKS
        self.conditional_true_block = ConditionalBlock(inputs=[self.cond])
        self.conditional_false_block = ConditionalBlock(inputs=[self.cond])
        self.output_table = ([], [])  # (true_outs, false_outs)

    def input(self, x):
        if self.status == IfElse.OUT_IF_ELSE_BLOCKS:
            raise ValueError("input must in true/false blocks")
        if id(x) not in self.input_table:
2645
            parent_block = self._parent_block()
Y
Yu Yang 已提交
2646
            out_true = parent_block.create_var(
2647 2648
                name=unique_name.generate_with_ignorable_key('ifelse_input' +
                                                             self.helper.name),
F
fengjiayi 已提交
2649
                dtype=x.dtype)
Y
Yu Yang 已提交
2650 2651

            out_false = parent_block.create_var(
2652 2653
                name=unique_name.generate_with_ignorable_key('ifelse_input' +
                                                             self.helper.name),
F
fengjiayi 已提交
2654
                dtype=x.dtype)
Y
Yu Yang 已提交
2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672
            parent_block.append_op(
                type='split_lod_tensor',
                inputs={
                    'X': x,
                    'Mask': self.cond,
                },
                outputs={'OutTrue': out_true,
                         'OutFalse': out_false},
                attrs={'level': 0})
            self.input_table[id(x)] = (out_true, out_false)
        else:
            out_true, out_false = self.input_table[id(x)]

        if self.status == IfElse.IN_IF_ELSE_TRUE_BLOCKS:
            return out_true
        else:
            return out_false

2673
    def _parent_block(self):
Y
Yu Yang 已提交
2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688
        current_block = self.helper.main_program.current_block()
        return self.helper.main_program.block(current_block.parent_idx)

    def true_block(self):
        return IfElseBlockGuard(True, self)

    def false_block(self):
        return IfElseBlockGuard(False, self)

    def output(self, *outs):
        if self.status == self.OUT_IF_ELSE_BLOCKS:
            raise ValueError("output can only be invoked in the sub-block")

        out_table = self.output_table[1 if self.status ==
                                      self.IN_IF_ELSE_TRUE_BLOCKS else 0]
2689
        parent_block = self._parent_block()
Y
Yu Yang 已提交
2690
        for each_out in outs:
2691 2692
            check_type(each_out, "each output", Variable,
                       "fluid.layers.IfElse.output")
Y
Yu Yang 已提交
2693 2694
            # create outside tensor
            outside_out = parent_block.create_var(
2695
                name=unique_name.generate_with_ignorable_key("_".join(
Y
Yu Yang 已提交
2696
                    [self.helper.name, 'output'])),
F
fengjiayi 已提交
2697
                dtype=each_out.dtype)
Y
Yu Yang 已提交
2698 2699 2700
            out_table.append(outside_out)

            # assign local var to outside
2701
            assign(input=each_out, output=outside_out)
Y
Yu Yang 已提交
2702 2703 2704 2705

    def __call__(self):
        if self.status != self.OUT_IF_ELSE_BLOCKS:
            raise ValueError("IfElse::__call__ must be out of sub-block")
2706
        false_len, true_len = list(map(len, self.output_table))
Y
Yu Yang 已提交
2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
        if false_len == 0 and true_len == 0:
            raise ValueError("Must invoke true_block/false_block before "
                             "__call__")
        elif false_len != true_len and false_len != 0 and true_len != 0:
            raise ValueError("The output side must be same")
        elif false_len == 0 or true_len == 0:
            return self.output_table[0 if false_len != 0 else 1]

        # else none of false_len/true_len is zero
        # merge together
        rlist = []
        for false_var, true_var in zip(*self.output_table):
            rlist.append(
                merge_lod_tensor(
                    in_true=true_var,
                    in_false=false_var,
                    mask=self.cond,
                    x=self.cond,
2725
                    level=0))
Y
Yu Yang 已提交
2726
        return rlist
2727 2728 2729


class DynamicRNN(object):
Y
yuyang18 已提交
2730
    """
2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742
    **Note: the input of this class should be LoDTensor which holds the
    information of variable-length sequences. If the input is fixed-length Tensor,
    please use StaticRNN (fluid.layers.** :ref:`api_fluid_layers_StaticRNN` **) for
    better performance.**

    DynamicRNN can process a minibatch of variable-length sequences.
    The length of each sample can be different and is recorded in LoD.
    In DynamicRNN, an input sequence will be unfolded into time steps and users
    can define how to process each time step in :code:`block()` .
    The total number of time steps is determined by the longest sequence.
    DynamicRNN will not pad all sequences to the same length, instead it will
    sort the sequences internally by the sequence length in descending order.
T
tianshuo78520a 已提交
2743
    The input sequences will be shrank because only sequences of which the
2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755
    length is larger than the time step will participate the remaining calculation.

    If defined :code:`drnn = DynamicRNN()`, then users can call :code:`drnn()`
    to obtain the result sequences. It is a LoDTensor gained by merging all
    time steps's output. When RNN's input sequence x meets :code:`x.lod_level == 1`,
    the output LoDTensor will have the same LoD with x. The result of :code:`drnn()`
    includes RNN's outputs of all time steps, users can call
    :ref:`api_fluid_layers_sequence_last_step` to extract the data of the last time step.

    Warning:
        Currently it is not supported to set :code:`is_sparse = True` of any
        layers defined within DynamicRNN's :code:`block` function.
Y
yuyang18 已提交
2756

2757 2758 2759 2760
    Args:
        name (str, optional): The default value is None.  Normally there is no
            need for user to set this property.  For more information,
            please refer to :ref:`api_guide_Name` .
2761 2762 2763 2764

    Examples:
        .. code-block:: python

2765
            import paddle.fluid as fluid
2766

2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792
            sentence = fluid.data(name='sentence', shape=[None, 32], dtype='float32', lod_level=1)
            encoder_proj = fluid.data(name='encoder_proj', shape=[None, 32], dtype='float32', lod_level=1)
            decoder_boot = fluid.data(name='boot', shape=[None, 10], dtype='float32')

            drnn = fluid.layers.DynamicRNN()
            with drnn.block():
                # Set sentence as RNN's input, each time step processes a word from the sentence
                current_word = drnn.step_input(sentence)
                # Set encode_proj as RNN's static input
                encoder_word = drnn.static_input(encoder_proj)
                # Initialize memory with boot_memory, which need reorder according to RNN's input sequences
                memory = drnn.memory(init=decoder_boot, need_reorder=True)
                fc_1 = fluid.layers.fc(input=encoder_word, size=30)
                fc_2 = fluid.layers.fc(input=current_word, size=30)
                decoder_inputs = fc_1 + fc_2
                hidden, _, _ = fluid.layers.gru_unit(input=decoder_inputs, hidden=memory, size=30)
                # Update memory with hidden
                drnn.update_memory(ex_mem=memory, new_mem=hidden)
                out = fluid.layers.fc(input=hidden, size=10, bias_attr=True, act='softmax')
                # Set hidden and out as RNN's outputs
                drnn.output(hidden, out)

            # Get RNN's result
            hidden, out = drnn()
            # Get RNN's result of the last time step
            last = fluid.layers.sequence_last_step(out)
Y
yuyang18 已提交
2793
    """
2794 2795 2796 2797
    BEFORE_RNN = 0
    IN_RNN = 1
    AFTER_RNN = 2

2798 2799
    def __init__(self, name=None):
        self.helper = LayerHelper('dynamic_rnn', name=name)
2800 2801 2802 2803
        self.status = DynamicRNN.BEFORE_RNN
        self.lod_rank_table = None
        self.max_seq_len = None
        self.step_idx = None
2804
        self.zero_idx = None
2805 2806 2807
        self.mem_dict = dict()
        self.output_array = []
        self.outputs = []
X
Xin Pan 已提交
2808
        self.cond = self.helper.create_variable_for_type_inference(dtype='bool')
2809 2810 2811 2812 2813
        self.cond.stop_gradient = False
        self.while_op = While(self.cond)
        self.input_array = []
        self.mem_link = []

2814
    def step_input(self, x, level=0):
Y
yuyang18 已提交
2815
        """
2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858
        This function is used to set sequence x as DynamicRNN's input.
        The maximum sequence length in x determines the number of time steps
        the RNN unit will be executed. DynamicRNN can take multiple inputs.
        When all inputs' :code:`lod_level` are 1, all inputs should hold the
        same LoD. When :code:`x.lod_level >= 2` , the input sequence will be
        unfold along specified level, and the slice of each time step is a
        LoDTensor whose lod_level is :code:`x.lod_level - level - 1` .
        In this case, the specified LoD level of multiple inputs should be the same.

        - Case 1:

        .. code-block:: text

            # input, where Si is slice data of shape [1, N]
            level = 0
            x.lod = [[2, 1, 3]]
            x.shape = [6, N]
            x.data = [[S0],
                      [S0],
                      [S1],
                      [S2],
                      [S2],
                      [S2]]

            # output
            # step 0, time step data of 3 sequences
            out.lod = [[]]
            out.shape = [3, N]
            out.data = [[S2],
                        [S0],
                        [S1]]

            # step 1, time step data of 2 sequences
            out.lod = [[]]
            out.shape = [2, N]
            out.data = [[S2],
                        [S0]]

            # step 2, time step data of 1 sequences
            out.lod = [[]]
            out.shape = [1, N]
            out.data = [[S2]]

H
haowang101779990 已提交
2859

Y
yuyang18 已提交
2860
        Args:
2861 2862 2863 2864 2865 2866 2867
            x (Variable): The input LoDTensor which holds information of a
                minibatch of variable-length sequences and should meet :code:`x.lod_level >= 1` .
                When RNN has multiple inputs, the first dimension should match
                across all inputs, but other shape components may differ.
                Optional data types are: bool, float16, float32, float64, int8, int16, int32, int64, uint8.
            level (int, optional): The level of lod used to split steps.
                It should be in range :math:`[0, x.lod\_level)` . The default value is 0.
Y
yuyang18 已提交
2868 2869

        Returns:
2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903
            Variable: The current time step in the input sequence. If there are :code:`num_sequences` \
                sequences in x whose length is larger than :code:`step_idx` , the returned Variable \
                will only hold the :code:`step_idx` -th time step of those `num_sequences` sequences. \
                The data type is the same as input. If :code:`x.lod_level == 1` , the return value is \
                a Tensor of shape :math:`\{num\_sequences, x.shape[1], ...\}` , or it will \
                be a variable-length LoDTensor.

        Raises:
            ValueError: When :code:`step_input()` is called outside :code:`block()` .
            TypeError: When x is not a Variable.

        Examples:
            ..  code-block:: python

                import paddle.fluid as fluid

                sentence = fluid.data(name='sentence', shape=[None, 1], dtype='int64', lod_level=1)
                embedding = fluid.layers.embedding(input=sentence, size=[65536, 32], is_sparse=True)

                drnn = fluid.layers.DynamicRNN()
                with drnn.block():
                    # Set embedding as RNN's input, each time step processes a word from the sentence
                    word = drnn.step_input(embedding)
                    # Initialize memory to a Tensor whose value is 0, shape=[batch_size, 200],
                    # where batch_size is the number of sequences in embedding.
                    memory = drnn.memory(shape=[200])
                    hidden = fluid.layers.fc(input=[word, memory], size=200, act='relu')
                    # Update memory to hidden
                    drnn.update_memory(ex_mem=memory, new_mem=hidden)
                    # Set hidden as RNN's output
                    drnn.output(hidden)

                # Get RNN's result
                rnn_output = drnn()
Y
yuyang18 已提交
2904
        """
2905 2906 2907
        self._assert_in_rnn_block_("step_input")
        if not isinstance(x, Variable):
            raise TypeError(
2908
                "step_input() can only take a Variable as its input.")
2909 2910 2911
        parent_block = self._parent_block_()
        if self.lod_rank_table is None:
            self.lod_rank_table = parent_block.create_var(
Y
Yu Yang 已提交
2912
                name=unique_name.generate('lod_rank_table'),
2913 2914 2915 2916 2917
                type=core.VarDesc.VarType.LOD_RANK_TABLE)
            self.lod_rank_table.stop_gradient = True
            parent_block.append_op(
                type='lod_rank_table',
                inputs={"X": x},
2918 2919
                outputs={"Out": self.lod_rank_table},
                attrs={"level": level})
2920
            self.max_seq_len = parent_block.create_var(
Y
Yu Yang 已提交
2921 2922
                name=unique_name.generate('dynamic_rnn_max_seq_len'),
                dtype='int64')
2923 2924 2925 2926 2927 2928 2929 2930 2931 2932
            self.max_seq_len.stop_gradient = False
            parent_block.append_op(
                type='max_sequence_len',
                inputs={'RankTable': self.lod_rank_table},
                outputs={"Out": self.max_seq_len})
            self.cond.stop_gradient = True
            parent_block.append_op(
                type='less_than',
                inputs={'X': self.step_idx,
                        'Y': self.max_seq_len},
J
JiayiFeng 已提交
2933 2934
                outputs={'Out': self.cond},
                attrs={'force_cpu': True})
2935 2936

        input_array = parent_block.create_var(
Y
Yu Yang 已提交
2937
            name=unique_name.generate('dynamic_rnn_input_array'),
2938 2939 2940 2941 2942 2943 2944 2945
            type=core.VarDesc.VarType.LOD_TENSOR_ARRAY,
            dtype=x.dtype)
        self.input_array.append((input_array, x.dtype))
        parent_block.append_op(
            type='lod_tensor_to_array',
            inputs={'X': x,
                    'RankTable': self.lod_rank_table},
            outputs={'Out': input_array})
2946
        return array_read(array=input_array, i=self.step_idx)
2947

Y
yangyaming 已提交
2948
    def static_input(self, x):
Y
yuyang18 已提交
2949
        """
2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022
        This function is used to set x as DynamicRNN's static input. It is optional.

        - Case 1, set static input with LoD

        .. code-block:: text

            # RNN's input is the same as the case listed in step_input
            # static input, where Si is slice data of shape [1, M]
            x.lod = [[3, 1, 2]]
            x.shape = [6, M]
            x.data = [[S0],
                      [S0],
                      [S0],
                      [S1],
                      [S2],
                      [S2]]

            # step 0, batch data corresponding to the 3 input sequences
            out.lod = [[2, 3, 1]]
            out.shape = [6, M]
            out.data = [[S2],
                        [S2],
                        [S0],
                        [S0],
                        [S0],
                        [S1]]

            # step 1, batch data corresponding to the 2 input sequences
            out.lod = [[2, 3]]
            out.shape = [5, M]
            out.data = [[S2],
                        [S2],
                        [S0],
                        [S0],
                        [S0]]

            # step 2, batch data corresponding to the 1 input sequences
            out.lod = [[2]]
            out.shape = [2, M]
            out.data = [[S2],
                        [S2]]


        - Case 2, set static input without LoD

        .. code-block:: text

            # RNN's input is the same as the case listed in step_input
            # static input, where Si is slice data of shape [1, M]
            x.lod = [[]]
            x.shape = [3, M]
            x.data = [[S0],
                      [S1],
                      [S2]]

            # step 0, batch data corresponding to the 3 input sequences
            out.lod = [[]]
            out.shape = [3, M]
            out.data = [[S2],
                        [S0],
                        [S1]]

            # step 1, batch data corresponding to the 2 input sequences
            out.lod = [[]]
            out.shape = [2, M]
            out.data = [[S2],
                        [S0]]

            # step 2, batch data corresponding to the 1 input sequences
            out.lod = [[]]
            out.shape = [1, M]
            out.data = [[S2]]

H
haowang101779990 已提交
3023

Y
yuyang18 已提交
3024
        Args:
3025 3026 3027 3028
            x (Variable): The static input LoDTensor which should hold the same number of sequences
                as RNN's input (the input LoDTensor set by :code:`step_input()` ). If the LoD is None,
                the input x will be treated as a minibatch with :code:`x.shape[0]` sequences of length 1.
                Optional data types are: bool, float16, float32, float64, int8, int16, int32, int64, uint8.
Y
yuyang18 已提交
3029 3030

        Returns:
T
tianshuo78520a 已提交
3031
            Variable: The input LoDTensor after sorted and shrank. If there are :code:`num_sequences` \
3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042
                sequences in RNN's input LoDTensor whose length is larger than :code:`step_idx` , \
                the static input Tensor will be sorted to the same order as RNN's input and \
                will only retain data corresponding to those :code:`num_sequences` sequences. \
                The data type is the same as input. If :code:`x.lod == None` , the return value is \
                a Tensor of shape :math:`\{num\_sequences, x.shape[1], ...\}` , or it will \
                be a variable-length LoDTensor.

        Raises:
            ValueError: When :code:`static_input()` is called outside :code:`block()` .
            TypeError: When x is not a Variable.
            RuntimeError: When :code:`static_input()` is called before :code:`step_input()` .
3043 3044 3045 3046

        Examples:
            .. code-block:: python

3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072
                import paddle.fluid as fluid

                sentence = fluid.data(name='sentence', shape=[None, 32], dtype='float32', lod_level=1)
                encoder_proj = fluid.data(name='encoder_proj', shape=[None, 32], dtype='float32', lod_level=1)
                decoder_boot = fluid.data(name='boot', shape=[None, 10], dtype='float32')

                drnn = fluid.layers.DynamicRNN()
                with drnn.block():
                    # Set sentence as RNN's input, each time step processes a word from the sentence
                    current_word = drnn.step_input(sentence)
                    # Set encode_proj as RNN's static input
                    encoder_word = drnn.static_input(encoder_proj)
                    # Initialize memory with boot_memory, which need reorder according to RNN's input sequences
                    memory = drnn.memory(init=decoder_boot, need_reorder=True)
                    fc_1 = fluid.layers.fc(input=encoder_word, size=30)
                    fc_2 = fluid.layers.fc(input=current_word, size=30)
                    decoder_inputs = fc_1 + fc_2
                    hidden, _, _ = fluid.layers.gru_unit(input=decoder_inputs, hidden=memory, size=30)
                    # Update memory with hidden
                    drnn.update_memory(ex_mem=memory, new_mem=hidden)
                    out = fluid.layers.fc(input=hidden, size=10, bias_attr=True, act='softmax')
                    # Set out as RNN's output
                    drnn.output(out)

                # Get RNN's result
                rnn_output = drnn()
Y
yuyang18 已提交
3073
        """
Y
yangyaming 已提交
3074 3075 3076 3077 3078 3079 3080 3081 3082
        self._assert_in_rnn_block_("static_input")
        if not isinstance(x, Variable):
            raise TypeError(
                "static_input() can only take a Variable as its input")
        if self.lod_rank_table is None:
            raise RuntimeError(
                "static_input() must be called after step_input().")
        parent_block = self._parent_block_()
        x_reordered = parent_block.create_var(
Y
Yu Yang 已提交
3083
            name=unique_name.generate("dynamic_rnn_static_input_reordered"),
Y
yangyaming 已提交
3084 3085 3086 3087 3088 3089 3090 3091 3092
            type=core.VarDesc.VarType.LOD_TENSOR,
            dtype=x.dtype)
        parent_block.append_op(
            type='reorder_lod_tensor_by_rank',
            inputs={'X': [x],
                    'RankTable': [self.lod_rank_table]},
            outputs={'Out': [x_reordered]})
        return shrink_memory(x_reordered, self.step_idx, self.lod_rank_table)

S
rename  
sneaxiy 已提交
3093
    @signature_safe_contextmanager
3094
    def block(self):
Y
yuyang18 已提交
3095
        """
3096 3097 3098 3099 3100 3101
        The function is used to list the operations executed during
        each time step in RNN. The operation list will be executed :code:`max_sequence_len`
        times (where :code:`max_sequence_len` is the maximum length of RNN's input sequences).

        Raises:
            ValueError: When :code:`block()` is called multi-times.
Y
yuyang18 已提交
3102
        """
3103 3104
        if self.status != DynamicRNN.BEFORE_RNN:
            raise ValueError("rnn.block() can only be invoke once")
3105 3106
        self.step_idx = fill_constant(
            shape=[1], dtype='int64', value=0, force_cpu=True)
3107 3108 3109 3110
        self.step_idx.stop_gradient = False
        self.status = DynamicRNN.IN_RNN
        with self.while_op.block():
            yield
3111
            increment(x=self.step_idx, value=1.0, in_place=True)
3112 3113

            for new_mem, mem_array in self.mem_link:
3114 3115
                array_write(x=new_mem, i=self.step_idx, array=mem_array)

J
JiayiFeng 已提交
3116 3117 3118 3119 3120
            less_than(
                x=self.step_idx,
                y=self.max_seq_len,
                force_cpu=True,
                cond=self.cond)
3121 3122 3123 3124 3125

        self.status = DynamicRNN.AFTER_RNN
        for each_array in self.output_array:
            self.outputs.append(
                array_to_lod_tensor(
3126
                    x=each_array, table=self.lod_rank_table))
3127 3128

    def __call__(self, *args, **kwargs):
Y
yuyang18 已提交
3129
        """
T
tianshuo78520a 已提交
3130
        This function is used to get the output  sequences of DynamicRNN.
3131 3132 3133 3134 3135 3136 3137 3138 3139

        Args:
            None

        Returns:
            Variable or Variable list: RNN's output sequences.

        Raises:
            ValueError: When :code:`__call__()` is called before :code:`block()` .
Y
yuyang18 已提交
3140
        """
3141
        if self.status != DynamicRNN.AFTER_RNN:
3142 3143
            raise ValueError(("Output of the dynamic RNN can only be visited "
                              "outside the rnn block."))
3144 3145 3146 3147 3148
        if len(self.outputs) == 1:
            return self.outputs[0]
        else:
            return self.outputs

3149 3150 3151 3152 3153 3154
    def memory(self,
               init=None,
               shape=None,
               value=0.0,
               need_reorder=False,
               dtype='float32'):
Y
yuyang18 已提交
3155
        """
3156 3157 3158
        Create a memory Variable for DynamicRNN to deliver data cross time steps.
        It can be initialized by an existing Tensor or a constant Tensor of given
        dtype and shape.
Y
yuyang18 已提交
3159

3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171
        Args:
            init (Variable, optional): LoDTensor used to initialize the memory.
                If init is not None, it should hold the same number of sequences
                as RNN's input (the input LoDTensor set by :code:`step_input()` )
                and the memory will be initialized to it. If init's LoD is None,
                it will be treated as a minibatch with :code:`init.shape[0]` sequences
                of length 1. The default value is None.
            shape (list|tuple, optional): When init is None, it is used to specify
                the memory's shape. Note that the shape does not include the batch_size.
                If setting shape to :math:`\{D_1, D_2, ...\}` , the shape of memory Tensor
                will be :math:`\{batch\_size, D_1, D_2, ...\}` , where batch_size is
                determined by RNN's input sequences. The default value is None.
T
tianshuo78520a 已提交
3172
            value (float, optional): When init is None, it is used as initialized value
3173 3174
                of memory. The default value is 0.0.
            need_reorder (bool, optional): When init is not None, it determines whether
T
tianshuo78520a 已提交
3175
                the memory needs to reorder like the RNN's input sequences. It should be
3176 3177 3178 3179 3180 3181 3182
                set to True when the initialized memory depends on the order of input samples.
                The default value is False.
            dtype (str|numpy.dtype, optional): When init is None, it is used to set the
                data type of memory. The default value is "float32". Optional data types
                are: "float32", "float64", "int32", "int64".

        Returns:
T
tianshuo78520a 已提交
3183
            Variable: The memory LoDTensor after shrank.  If there are :code:`num_sequences` \
3184
                sequences in RNN's input LoDTensor whose length is larger than :code:`step_idx` , \
T
tianshuo78520a 已提交
3185
                the memory Tensor also need to be shrank and will only retain data \
3186 3187 3188 3189 3190 3191
                corresponding to those :code:`num_sequences` sequences.

        Raises:
            ValueError: When :code:`memory()` is called outside :code:`block()` .
            TypeError: When init is set and is not a Variable.
            ValueError: When :code:`memory()` is called before :code:`step_input()` .
Y
yuyang18 已提交
3192

3193 3194 3195
        Examples:
            .. code-block:: python

3196
                import paddle.fluid as fluid
3197

3198 3199
                sentence = fluid.data(name='sentence', shape=[None, 32], dtype='float32', lod_level=1)
                boot_memory = fluid.data(name='boot', shape=[None, 10], dtype='float32')
3200

3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211
                drnn = fluid.layers.DynamicRNN()
                with drnn.block():
                    # Set sentence as RNN's input, each time step processes a word from the sentence
                    word = drnn.step_input(sentence)
                    # Initialize memory with boot_memory, which need reorder according to RNN's input sequences
                    memory = drnn.memory(init=boot_memory, need_reorder=True)
                    hidden = fluid.layers.fc(input=[word, memory], size=10, act='tanh')
                    # Update memory with hidden
                    drnn.update_memory(ex_mem=memory, new_mem=hidden)
                    # Set hidden as RNN's output
                    drnn.output(hidden)
Y
yuyang18 已提交
3212

3213 3214
                # Get RNN's result
                rnn_output = drnn()
Y
yuyang18 已提交
3215 3216


3217 3218
        Examples:
            .. code-block:: python
Y
yuyang18 已提交
3219

3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238
                import paddle.fluid as fluid

                sentence = fluid.data(name='sentence', shape=[None, 32], dtype='float32', lod_level=1)

                drnn = fluid.layers.DynamicRNN()
                with drnn.block():
                    # Set sentence as RNN's input, each time step processes a word from the sentence
                    word = drnn.step_input(sentence)
                    # Initialize memory to a Tensor whose value is 0, shape=[batch_size, 10],
                    # where batch_size is the number of sequences in sentence.
                    memory = drnn.memory(shape=[10], dtype='float32', value=0)
                    hidden = fluid.layers.fc(input=[word, memory], size=10, act='tanh')
                    # Update memory with hidden
                    drnn.update_memory(ex_mem=memory, new_mem=hidden)
                    # Set hidden as RNN's output
                    drnn.output(hidden)

                # Get RNN's result
                rnn_output = drnn()
Y
yuyang18 已提交
3239
        """
3240
        self._assert_in_rnn_block_('memory')
3241
        self._init_zero_idx_()
3242 3243 3244 3245 3246
        if init is not None:
            if not isinstance(init, Variable):
                raise TypeError(
                    "The input arg `init` of memory() must be a Variable")
            parent_block = self._parent_block_()
3247 3248 3249 3250 3251 3252 3253 3254
            init_tensor = init
            if need_reorder == True:
                if self.lod_rank_table is None:
                    raise ValueError(
                        'If set need_reorder to True, make sure step_input be '
                        'invoked before '
                        'memory(init=init, need_reordered=True, ...).')
                init_reordered = parent_block.create_var(
Y
Yu Yang 已提交
3255
                    name=unique_name.generate('dynamic_rnn_mem_init_reordered'),
3256 3257 3258 3259 3260 3261 3262 3263 3264 3265
                    type=core.VarDesc.VarType.LOD_TENSOR,
                    dtype=init.dtype)
                parent_block.append_op(
                    type='reorder_lod_tensor_by_rank',
                    inputs={
                        'X': [init_tensor],
                        'RankTable': [self.lod_rank_table]
                    },
                    outputs={'Out': [init_reordered]})
                init_tensor = init_reordered
3266
            mem_array = parent_block.create_var(
Y
Yu Yang 已提交
3267
                name=unique_name.generate('dynamic_rnn_mem_array'),
3268 3269 3270 3271
                type=core.VarDesc.VarType.LOD_TENSOR_ARRAY,
                dtype=init.dtype)
            parent_block.append_op(
                type='write_to_array',
3272
                inputs={'X': init_tensor,
3273 3274
                        'I': self.zero_idx},
                outputs={'Out': mem_array})
3275
            retv = array_read(array=mem_array, i=self.step_idx)
3276
            retv = shrink_memory(
3277
                x=retv, i=self.step_idx, table=self.lod_rank_table)
3278 3279 3280 3281 3282 3283 3284 3285 3286
            self.mem_dict[retv.name] = mem_array
            return retv
        else:
            if len(self.input_array) == 0:
                raise ValueError(
                    "step_input should be invoked before memory(shape=..., value=...)"
                )
            parent_block = self._parent_block_()
            init = parent_block.create_var(
Y
Yu Yang 已提交
3287
                name=unique_name.generate('mem_init'), dtype=dtype)
3288
            arr, dtype = self.input_array[0]
Y
Yu Yang 已提交
3289 3290
            in0 = parent_block.create_var(
                name=unique_name.generate('in0'), dtype=dtype)
3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
            parent_block.append_op(
                type='read_from_array',
                inputs={'X': [arr],
                        'I': [self.zero_idx]},
                outputs={'Out': [in0]})
            parent_block.append_op(
                type='fill_constant_batch_size_like',
                inputs={'Input': [in0]},
                outputs={'Out': [init]},
                attrs={
                    'shape': [-1] + shape,
                    'value': float(value),
                    'dtype': init.dtype
                })
            return self.memory(init=init)

    def update_memory(self, ex_mem, new_mem):
Y
yuyang18 已提交
3308
        """
3309 3310
        Update the memory which need to be delivered across time steps.

Y
yuyang18 已提交
3311
        Args:
3312 3313 3314
            ex_mem (Variable): The memory data of previous time step.
            new_mem (Variable): The new memory data produced in current time step.
                The shape and data type of ex_mem and new_mem should be the same.
Y
yuyang18 已提交
3315 3316 3317

        Returns:
            None
3318 3319 3320 3321 3322 3323
        
        Raises:
            ValueError: When :code:`update_memory()` is called outside :code:`block()` .
            TypeError: When :code:`ex_mem` or :code:`new_mem` is not a Variable.
            ValueError: When :code:`ex_mem` is defined by :code:`memory()` .
            ValueError: When :code:`update_memory()` is called before :code:`step_input()` .
Y
yuyang18 已提交
3324
        """
3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341
        self._assert_in_rnn_block_('update_memory')
        if not isinstance(ex_mem, Variable):
            raise TypeError("The input arg `ex_mem` of update_memory() must "
                            "be a Variable")
        if not isinstance(new_mem, Variable):
            raise TypeError("The input arg `new_mem` of update_memory() must "
                            "be a Variable")

        mem_array = self.mem_dict.get(ex_mem.name, None)
        if mem_array is None:
            raise ValueError("Please invoke memory before update_memory")
        if self.lod_rank_table is None:
            raise ValueError("Please invoke step_input before update_memory")

        self.mem_link.append((new_mem, mem_array))

    def output(self, *outputs):
Y
yuyang18 已提交
3342
        """
3343
        This function is used to set :code:`outputs` as RNN's output.
Y
yuyang18 已提交
3344 3345

        Args:
3346 3347
            *outputs (Variable ...): The output Tensor. DynamicRNN can mark multiple
                Variables as its output.
Y
yuyang18 已提交
3348 3349 3350

        Returns:
            None
3351 3352 3353

        Raises:
            ValueError: When :code:`output()` is called outside :code:`block()` .
Y
yuyang18 已提交
3354
        """
3355 3356 3357 3358
        self._assert_in_rnn_block_('output')
        parent_block = self._parent_block_()
        for each in outputs:
            outside_array = parent_block.create_var(
3359
                name=unique_name.generate_with_ignorable_key("_".join(
3360 3361 3362 3363 3364 3365
                    [self.helper.name, "output_array", each.name])),
                type=core.VarDesc.VarType.LOD_TENSOR_ARRAY,
                dtype=each.dtype)
            array_write(x=each, i=self.step_idx, array=outside_array)
            self.output_array.append(outside_array)

3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381
    def _init_zero_idx_(self):
        if self.zero_idx is None:
            parent_block = self._parent_block_()
            self.zero_idx = parent_block.create_var(
                name=unique_name.generate('zero_idx'), dtype='int64')
            parent_block.append_op(
                type='fill_constant',
                inputs={},
                outputs={'Out': [self.zero_idx]},
                attrs={
                    'shape': [1],
                    'dtype': self.zero_idx.dtype,
                    'value': float(0),
                    'force_cpu': True
                })

3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393
    def _parent_block_(self):
        prog = self.helper.main_program
        parent_idx = prog.current_block().parent_idx
        assert parent_idx >= 0
        parent_block = prog.block(parent_idx)

        return parent_block

    def _assert_in_rnn_block_(self, method):
        if self.status != DynamicRNN.IN_RNN:
            raise ValueError("{0} can only be invoked inside rnn block.".format(
                method))
Y
Yang Yu 已提交
3394 3395


L
liym27 已提交
3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424
def switch_case(branch_index, branch_fns, default=None, name=None):
    '''
    This operator is like a C++ switch/case statement.

    Args:
        branch_index(Variable): A Tensor with shape [1] to specify which branch to execute. The data type is ``int32``, ``int64`` or ``uint8``.
        branch_fns(dict|list|tuple): If it's a list or tuple, the elements in it could be pairs of (int, callable) or simple callables whose actual index will be used as the index of callable. If it's a dict, its key is a python integer and the value is a callable. All callables return the same structure of Tensors.
        default(callable, optional): Callable that returns a structure of Tensors.
        name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`.

    Returns:
        Variable|list(Variable): Tensors returned by the callable specified by ``branch_index`` in ``branch_fns``,
        or Tensors returned by ``default`` if ``default`` is not None and no index matches in ``branch_fns``,
        or Tensors returned by the callable with the max index in ``branch_fns`` if ``default`` is None and no index matches in ``branch_fns``.

    Raises:
        TypeError: If the type of ``branch_index`` is not Variable.
        TypeError: If the data type of ``branch_index`` is not ``int32``, ``int64`` or ``uint8``.
        TypeError: If the type of ``branch_fns`` is not dict, list or tuple.
        TypeError: If the elements of ``branch_fns`` is not 2-tuple.
        TypeError: If the first element of 2-tuple in ``branch_fns`` is not integer.
        ValueError: If the first element of 2-tuple in ``branch_fns`` is not unique.
        TypeError: If the second element of 2-tuple in ``branch_fns`` is not callable.
        TypeError: If ``default`` is not None but it is not callable.

    Examples:
        .. code-block:: python

            import paddle.fluid as fluid
3425 3426
            import paddle.fluid.layers as layers

L
liym27 已提交
3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437
            def fn_1():
                return layers.fill_constant(shape=[1, 2], dtype='float32', value=1)

            def fn_2():
                return layers.fill_constant(shape=[2, 2], dtype='int32', value=2)

            def fn_3():
                return layers.fill_constant(shape=[3], dtype='int32', value=3)

            main_program = fluid.default_startup_program()
            startup_program = fluid.default_main_program()
3438
            with fluid.program_guard(main_program, startup_program):
L
liym27 已提交
3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457
                index_1 = layers.fill_constant(shape=[1], dtype='int32', value=1)
                index_2 = layers.fill_constant(shape=[1], dtype='int32', value=2)

                out_1 = layers.switch_case(
                    branch_index=index_1,
                    branch_fns={1: fn_1, 2: fn_2},
                    default=fn_3)

                out_2 = layers.switch_case(
                    branch_index=index_2,
                    branch_fns=[(1, fn_1), (2, fn_2)],
                    default=fn_3)

                # Argument default is None and no index matches. fn_3 will be called because of the max index 7.
                out_3 = layers.switch_case(
                    branch_index=index_2,
                    branch_fns=[(0, fn_1), (4, fn_2), (7, fn_3)])

                exe = fluid.Executor(fluid.CPUPlace())
3458
                res_1, res_2, res_3 = exe.run(main_program, fetch_list=[out_1, out_2, out_3])
L
liym27 已提交
3459 3460 3461 3462 3463 3464 3465 3466
                print(res_1)  # [[1. 1.]]
                print(res_2)  # [[2 2] [2 2]]
                print(res_3)  # [3 3 3]
    '''
    helper = LayerHelper('switch_case', **locals())

    def _check_args(branch_index, branch_fns, default):

3467 3468
        check_variable_and_dtype(branch_index, 'branch_index',
                                 ['uint8', 'int32', 'int64'], 'switch_case')
L
liym27 已提交
3469 3470 3471 3472

        if convert_dtype(branch_index.dtype) != "int64":
            branch_index = cast(branch_index, "int64")

3473
        check_type(branch_fns, 'branch_fns', (list, tuple, dict), 'switch_case')
L
liym27 已提交
3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485

        branch_fns = branch_fns.items() if isinstance(branch_fns,
                                                      dict) else branch_fns

        branch_fns = list(enumerate(branch_fns)) if all(
            callable(fn) for fn in branch_fns) else branch_fns

        keys_of_fns = []
        for index_fn_pair in branch_fns:
            if not isinstance(index_fn_pair, tuple):
                raise TypeError(
                    _error_message("The elements' type", "branch_fns",
3486
                                   "switch_case", tuple, type(branch_fns)))
L
liym27 已提交
3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498

            if len(index_fn_pair) != 2:
                raise TypeError(
                    _error_message("The tuple's size", "branch_fns",
                                   "switch_case", "2",
                                   str(len(index_fn_pair)) + "-tuple"))

            key, fn = index_fn_pair

            if not isinstance(key, int):
                raise TypeError(
                    _error_message("The key's type", "branch_fns",
3499
                                   "switch_case", int, type(key)))
L
liym27 已提交
3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536

            if key in keys_of_fns:
                raise ValueError(
                    "The key in 'branch_fns' must be unique, but '{}' appears more than once.".
                    format(key))
            else:
                keys_of_fns.append(key)

            if not callable(fn):
                raise TypeError(
                    _error_message("The type of function for key {}".format(
                        key), "branch_fns", "switch_case", "callable", type(
                            fn)))

        if default is None:
            default = sorted(branch_fns)[-1][1]
            branch_fns = sorted(branch_fns)[:-1]
        elif not callable(default):
            raise TypeError("The default in Op(case) must be callable.")

        pred_fn_pairs = []
        for index, fn in branch_fns:
            new_index = fill_constant(shape=[1], dtype="int64", value=index)
            pred = equal(branch_index, new_index)
            pred_fn_pairs.append((pred, fn))

        return pred_fn_pairs, default

    pred_fn_pairs, default = _check_args(branch_index, branch_fns, default)
    false_fn = default
    for pred, true_fn in pred_fn_pairs:
        false_fn = partial(cond, pred=pred, true_fn=true_fn, false_fn=false_fn)

    final_fn = false_fn
    return final_fn()


3537
@templatedoc()
Y
Yang Yu 已提交
3538
def reorder_lod_tensor_by_rank(x, rank_table):
3539 3540 3541 3542
    """
    ${comment}

    Args:
3543 3544
        x(${x_type}): ${x_comment}.
        rank_table(${rank_table_type}): ${rank_table_comment}.
3545 3546
    
    Returns:
3547
        out(${out_type}): ${out_comment}.
3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560

    Examples:
        .. code-block:: python

          import paddle.fluid as fluid
          data_desc = (['input', [9], 0], ['ref', [5], 1])
          data = fluid.layers.data(name=data_desc[0][0], shape=data_desc[0][1])
          rank_data = fluid.layers.data(name=data_desc[1][0], shape=data_desc[1][1])
          table = fluid.layers.control_flow.lod_rank_table(rank_data)
          new_data = fluid.layers.reorder_lod_tensor_by_rank(
                           x=data, rank_table=table)

    """
3561 3562 3563 3564 3565 3566 3567

    check_type(x, 'x', (Variable), 'reorder_lod_tensor_by_rank')
    check_type(rank_table, 'rank_table', (Variable),
               'reorder_lod_tensor_by_rank')
    if rank_table.type != core.VarDesc.VarType.LOD_RANK_TABLE:
        raise TypeError("The type of rank_table should be LOD_RANK_TABLE.")

Y
Yang Yu 已提交
3568 3569
    helper = LayerHelper('reorder_lod_tensor_by_rank', **locals())

X
Xin Pan 已提交
3570
    out = helper.create_variable_for_type_inference(dtype=x.dtype)
Y
Yang Yu 已提交
3571 3572 3573 3574 3575 3576
    helper.append_op(
        type='reorder_lod_tensor_by_rank',
        inputs={'X': [x],
                'RankTable': [rank_table]},
        outputs={'Out': [out]})
    return out
3577 3578


3579
def is_empty(x, cond=None):
3580
    """
F
fengjiayi 已提交
3581
    Test whether a Variable is empty.
3582 3583

    Args:
F
fengjiayi 已提交
3584
        x (Variable): The Variable to be tested.
3585 3586
        cond (Variable, optional): Output parameter. Default: None. If this parameter is given, it
                              saves the test result of given 'x'.
3587 3588

    Returns:
F
fengjiayi 已提交
3589
        Variable: A bool scalar. True if 'x' is an empty Variable.
3590 3591 3592

    Raises:
        TypeError: If input cond is not a variable, or cond's dtype is
F
fengjiayi 已提交
3593
                   not bool.
3594 3595 3596 3597

    Examples:
        .. code-block:: python

3598 3599
          import paddle.fluid as fluid
          input = fluid.layers.data(name="input", shape=[4, 32, 32], dtype="float32")
F
fengjiayi 已提交
3600 3601
          res = fluid.layers.is_empty(x=input)
          # or:
3602 3603
          # fluid.layers.is_empty(x=input, cond=res)

3604 3605 3606
    """
    helper = LayerHelper("is_empty", **locals())
    if cond is None:
X
Xin Pan 已提交
3607
        cond = helper.create_variable_for_type_inference(dtype='bool')
3608 3609 3610 3611 3612 3613 3614 3615 3616
        cond.stop_gradient = True
    elif not isinstance(cond, Variable):
        raise TypeError("cond takes a variable")
    elif cond.dtype != 'bool':
        raise TypeError("The data type of cond must be bool")

    helper.append_op(
        type='is_empty', inputs={'X': [x]}, outputs={'Out': [cond]})
    return cond