framework.py 18.1 KB
Newer Older
Y
Yu Yang 已提交
1
import paddle.v2.framework.core as core
F
fengjiayi 已提交
2
import paddle.v2.framework.proto.framework_pb2 as framework_pb2
Y
Yu Yang 已提交
3
import collections
Y
Yu Yang 已提交
4
import numpy as np
Y
Yu Yang 已提交
5
import copy
Y
Yu Yang 已提交
6

Y
Yu Yang 已提交
7
__all__ = ['Block', 'Variable', 'Program', 'Operator']
Y
Yu Yang 已提交
8 9 10


class Variable(object):
Y
Yu Yang 已提交
11 12
    def __init__(self,
                 block,
Y
Yu Yang 已提交
13
                 type=core.VarDesc.VarType.LOD_TENSOR,
Y
Yu Yang 已提交
14 15 16 17
                 name=None,
                 shape=None,
                 dtype=None,
                 lod_level=None,
Q
QI JUN 已提交
18
                 persistable=None,
Y
Yu Yang 已提交
19
                 **kwargs):
Y
Yu Yang 已提交
20 21 22 23
        self.block = block

        if name is None:
            name = Variable._unique_var_name_()
D
Dong Zhihong 已提交
24 25 26 27
        is_new_var = False
        self.desc = self.block.desc.find_var(name)

        if self.desc is None:
D
dongzhihong 已提交
28
            self.desc = self.block.desc.var(name)
Y
Yu Yang 已提交
29
            is_new_var = True
Y
Yu Yang 已提交
30

Y
Yu Yang 已提交
31 32 33 34 35 36 37 38
        if is_new_var:
            self.desc.set_type(type)
        elif self.desc.type() != type:
            raise ValueError("Variable {0} has been created before. The "
                             "previous type is {1}; the new type is {2}. They"
                             " are not matched".format(self.name,
                                                       self.desc.type(), type))

Y
Yu Yang 已提交
39
        if shape is not None:
Y
Yu Yang 已提交
40
            if is_new_var:
41
                self.desc.set_shape(shape)
Y
Yu Yang 已提交
42 43 44 45 46 47 48 49
            else:
                old_shape = self.shape
                shape = tuple(shape)
                if shape != old_shape:
                    raise ValueError(
                        "Variable {0} has been created before. the previous "
                        "shape is {1}; the new shape is {2}. They are not "
                        "matched.".format(self.name, old_shape, shape))
Y
Yu Yang 已提交
50
        if dtype is not None:
Y
Yu Yang 已提交
51 52
            if not isinstance(dtype, core.DataType):
                dtype = Variable._convert_np_dtype_to_dtype_(dtype)
Y
Yu Yang 已提交
53
            if is_new_var:
54
                self.desc.set_data_type(dtype)
Y
Yu Yang 已提交
55
            else:
Q
QI JUN 已提交
56 57
                old_dtype = self.data_type
                if dtype != old_dtype:
Y
Yu Yang 已提交
58 59 60 61 62
                    raise ValueError("Variable {0} has been created before. "
                                     "The previous data type is {1}; the new "
                                     "data type is {2}. They are not "
                                     "matched.".format(self.name, old_dtype,
                                                       dtype))
Y
Yu Yang 已提交
63 64

        if lod_level is not None:
Y
Yu Yang 已提交
65
            if is_new_var:
66
                self.desc.set_lod_level(lod_level)
Y
Yu Yang 已提交
67 68 69 70 71 72 73
            else:
                if lod_level != self.lod_level:
                    raise ValueError("Variable {0} has been created before. "
                                     "The previous lod_level is {1}; the new "
                                     "lod_level is {2}. They are not "
                                     "matched".format(self.name, self.lod_level,
                                                      lod_level))
74 75 76 77 78 79 80 81 82 83 84
        if persistable is not None:
            if is_new_var:
                self.desc.set_persistable(persistable)
            else:
                if persistable != self.persistable:
                    raise ValueError(
                        "Variable {0} has been created before."
                        "The previous persistable is {1}; the new "
                        "persistable is {2}. They are not matched".format(
                            self.name, self.persistable, persistable))

Y
Yu Yang 已提交
85
        self.block.vars[name] = self
Y
Yu Yang 已提交
86
        self.op = None
Y
Yu Yang 已提交
87

88 89 90 91 92 93 94
    def __str__(self):
        protostr = self.desc.serialize_to_string()
        proto = framework_pb2.VarDesc.FromString(str(protostr))
        return proto.__str__()

    __repr__ = __str__

95 96 97 98
    @property
    def persistable(self):
        return self.desc.persistable()

Y
Yu Yang 已提交
99 100
    @property
    def name(self):
101
        return self.desc.name()
Y
Yu Yang 已提交
102 103 104 105

    @property
    def shape(self):
        # convert to tuple, make it as same as numpy API.
106
        return tuple(self.desc.shape())
Y
Yu Yang 已提交
107 108 109

    @property
    def data_type(self):
110
        return self.desc.data_type()
Y
Yu Yang 已提交
111 112 113

    @property
    def lod_level(self):
114
        return self.desc.lod_level()
Y
Yu Yang 已提交
115

Y
Yu Yang 已提交
116 117 118 119
    @property
    def type(self):
        return self.desc.type()

Y
Yu Yang 已提交
120 121
    @staticmethod
    def _unique_var_name_():
122 123 124
        prefix = "_generated_var"
        uid = core.unique_integer(prefix)  # unique during whole process.
        return "_".join([prefix, str(uid)])
Y
Yu Yang 已提交
125

Y
Yu Yang 已提交
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
    @staticmethod
    def _convert_np_dtype_to_dtype_(np_dtype):
        dtype = np.dtype(np_dtype)
        if dtype == np.float32:
            return core.DataType.FP32
        elif dtype == np.float64:
            return core.DataType.FP64
        elif dtype == np.float16:
            return core.DataType.FP16
        elif dtype == np.int32:
            return core.DataType.INT32
        elif dtype == np.int16:
            return core.DataType.INT16
        elif dtype == np.int64:
            return core.DataType.INT64
        elif dtype == np.bool:
            return core.DataType.BOOL
        else:
            raise ValueError("Not supported numpy dtype " + str(dtype))

Y
Yu Yang 已提交
146

F
fengjiayi 已提交
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
def get_all_op_protos():
    """
    Get all registered op proto from PaddlePaddle C++ end.
    :return: A list of registered OpProto.
    """
    protostrs = core.get_all_op_protos()
    ret_values = []
    for pbstr in protostrs:
        op_proto = framework_pb2.OpProto.FromString(str(pbstr))
        ret_values.append(op_proto)
    return ret_values


class OpProtoHolder(object):
    @classmethod
    def instance(cls):
        if not hasattr(cls, '_instance'):
            cls._instance = cls()
        return cls._instance

    def __init__(self):
        assert not hasattr(
            self.__class__,
            '_instance'), 'Please use `instance()` to get OpProtoHolder opject!'
        op_protos = get_all_op_protos()
        self.op_proto_map = {}
        for proto in op_protos:
            self.op_proto_map[proto.type] = proto

    def get_op_proto(self, type):
Y
Yu Yang 已提交
177 178
        if type not in self.op_proto_map:
            raise ValueError("Operator \"%s\" has not been registered." % type)
F
fengjiayi 已提交
179 180 181
        return self.op_proto_map[type]


Y
Yu Yang 已提交
182 183 184
class Operator(object):
    def __init__(self,
                 block,
Y
Yu Yang 已提交
185
                 desc,
Y
Yu Yang 已提交
186 187 188 189 190
                 type=None,
                 inputs=None,
                 outputs=None,
                 attrs=None):
        self.block = block
Y
Yu Yang 已提交
191
        self.desc = desc
F
fengjiayi 已提交
192 193 194 195 196
        if len(self.desc.type()) != 0:
            return
        if type is None:
            raise ValueError(
                "`type` to initilized an Operator can not be None.")
F
Update  
fengjiayi 已提交
197
        self.desc.set_type(type)
F
fengjiayi 已提交
198
        proto = OpProtoHolder.instance().get_op_proto(type)
199

Y
Yang Yang(Tony) 已提交
200 201 202 203 204
        def find_name(var_list, name):
            for var_name in var_list:
                if var_name == name:
                    return True
            return False
Q
QI JUN 已提交
205

Y
Yang Yang(Tony) 已提交
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
        if inputs is not None:
            for in_proto in proto.inputs:
                found = find_name(inputs, in_proto.name)
                assert found or in_proto.dispensable, "Input {} not found".format(
                    in_proto.name)

                if found:
                    in_argus = inputs[in_proto.name]
                    if not isinstance(in_argus, list):
                        in_argus = [in_argus]
                    if not in_proto.duplicable and len(in_argus) > 1:
                        raise ValueError(
                            "Input %s expects only one input, but %d are given."
                            % (in_proto.name, len(in_argus)))
                    in_argu_names = []
                    for argu in in_argus:
                        in_argu_names.append(argu.name)
                    self.desc.set_input(in_proto.name, in_argu_names)
                else:
                    self.desc.set_input(in_proto.name, [])
F
Update  
fengjiayi 已提交
226

Y
Yu Yang 已提交
227
        if outputs is not None:
228 229 230 231 232 233 234 235 236 237 238 239
            given = set()
            need = set()
            for n in outputs:
                given.add(n)
            for m in proto.outputs:
                need.add(m.name)
            if not given == need:
                raise ValueError(
                    "Incorrect setting for output(s) of operator \"%s\". Need: [%s] Given: [%s]"
                    % (type, ", ".join(str(e) for e in need), ", ".join(
                        str(e) for e in given)))

F
fengjiayi 已提交
240
            for out_proto in proto.outputs:
F
Update  
fengjiayi 已提交
241 242 243 244 245 246 247 248 249
                out_argus = outputs[out_proto.name]
                if not isinstance(out_argus, list):
                    out_argus = [out_argus]
                if not out_proto.duplicable and len(out_argus) > 1:
                    raise ValueError(
                        "Output %s expects only one output, but %d are given." %
                        (out_proto.name, len(out_argus)))
                out_argu_names = []
                for argu in out_argus:
F
fengjiayi 已提交
250
                    out_argu_names.append(argu.name)
F
fengjiayi 已提交
251
                    argu.op = self
F
Update  
fengjiayi 已提交
252 253
                self.desc.set_output(out_proto.name, out_argu_names)

Y
Yu Yang 已提交
254
        if attrs is not None:
255 256
            if not isinstance(attrs, dict):
                raise TypeError("'attrs' should be a dict.")
F
fengjiayi 已提交
257
            for attr in proto.attrs:
F
Update  
fengjiayi 已提交
258
                attr_name = attr.name
259
                if (not attr_name in attrs) or (attrs[attr_name] is None):
F
Update  
fengjiayi 已提交
260
                    continue
Y
Yang Yang(Tony) 已提交
261
                if isinstance(attrs[attr_name], Block):
F
Update  
fengjiayi 已提交
262
                    self.desc.set_block_attr(attr_name, attrs[attr_name].desc)
Y
Yang Yang(Tony) 已提交
263 264
                else:
                    self.desc.set_attr(attr_name, attrs[attr_name])
Y
Yu Yang 已提交
265

266
        self.desc.check_attrs()
Y
Yu Yang 已提交
267
        no_kernel_op_set = {'feed', 'fetch', 'save', 'load'}
268
        if type not in no_kernel_op_set:
Q
QI JUN 已提交
269
            self.desc.infer_var_type(self.block.desc)
Y
Yu Yang 已提交
270
            self.desc.infer_shape(self.block.desc)
F
fengjiayi 已提交
271

272 273 274 275 276 277 278
    def __str__(self):
        protostr = self.desc.serialize_to_string()
        proto = framework_pb2.OpDesc.FromString(str(protostr))
        return proto.__str__()

    __repr__ = __str__

F
fengjiayi 已提交
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
    @property
    def type(self):
        return self.desc.type()

    def input(self, name):
        return self.desc.input(name)

    @property
    def input_names(self):
        return self.desc.input_names()

    def output(self, name):
        return self.desc.output(name)

    @property
    def output_names(self):
        return self.desc.output_names()

297 298 299 300 301 302 303 304
    @property
    def idx(self):
        for i, op in enumerate(self.block.ops):
            if op == self:
                return i
        raise ValueError(
            "Can't find op itself in it's block. It could be a bug of Paddle.")

F
fengjiayi 已提交
305 306 307 308 309 310 311 312 313 314 315 316
    def has_attr(self, name):
        return self.desc.has_attr(name)

    def attr_type(self, name):
        return self.desc.attr_type(name)

    @property
    def attr_names(self):
        return self.desc.attr_names()

    def attr(self, name):
        return self.desc.attr(name)
Y
Yu Yang 已提交
317

F
fengjiayi 已提交
318 319
    def block_attr(self, name):
        return self.desc.block_attr(name)
Y
Yu Yang 已提交
320 321


Y
Yu Yang 已提交
322 323
class Block(object):
    def __init__(self, program, idx):
Y
Yu Yang 已提交
324
        self.desc = program.desc.block(idx)
Y
Yu Yang 已提交
325
        self.vars = dict()  # var_name --> var
Y
Yu Yang 已提交
326
        self.ops = collections.deque()  # operator list
Y
Yu Yang 已提交
327 328
        self.program = program

329 330 331 332 333 334 335
    def __str__(self):
        protostr = self.desc.serialize_to_string()
        proto = framework_pb2.BlockDesc.FromString(str(protostr))
        return proto.__str__()

    __repr__ = __str__

Y
Yu Yang 已提交
336 337
    @property
    def parent_idx(self):
Y
Yu Yang 已提交
338
        return self.desc.parent
Y
Yu Yang 已提交
339 340 341

    @property
    def idx(self):
Y
Yu Yang 已提交
342
        return self.desc.id
Y
Yu Yang 已提交
343

Q
Qiao Longfei 已提交
344
    def var(self, name):
Y
Yu Yang 已提交
345 346 347 348
        if not isinstance(name, basestring):
            raise TypeError()
        v = self.vars.get(name, None)
        if v is None:
Q
Qiao Longfei 已提交
349
            raise ValueError("var %s not in this block" % name)
Y
Yu Yang 已提交
350
        return v
Q
Qiao Longfei 已提交
351 352 353 354

    def all_parameters(self):
        return {v for k, v in self.vars.iteritems() if isinstance(v, Parameter)}

Y
Yu Yang 已提交
355
    def create_var(self, *args, **kwargs):
Q
Qiao Longfei 已提交
356 357 358 359
        var = Variable(self, *args, **kwargs)
        if 'init_attr' in kwargs:
            self._prepend_initialize_ops_(var, kwargs['init_attr'])
        return var
Y
Yu Yang 已提交
360

Q
Qiao Longfei 已提交
361 362 363
    def has_var(self, name):
        return name in self.vars

Y
Yu Yang 已提交
364 365
    def create_parameter(self, *args, **kwargs):
        global_block = self.program.global_block()
Q
Qiao Longfei 已提交
366
        param = Parameter(global_block, *args, **kwargs)
Q
QI JUN 已提交
367 368
        if 'init_attr' in kwargs:
            self._prepend_initialize_ops_(param, kwargs['init_attr'])
Q
Qiao Longfei 已提交
369
        return param
Y
Yu Yang 已提交
370

Y
Yu Yang 已提交
371
    def append_op(self, *args, **kwargs):
Y
Yu Yang 已提交
372 373
        op_desc = self.desc.append_op()
        op = Operator(self, op_desc, *args, **kwargs)
Y
Yu Yang 已提交
374 375 376 377
        self.ops.append(op)
        return op

    def prepend_op(self, *args, **kwargs):
Y
Yu Yang 已提交
378 379
        op_desc = self.desc.prepend_op()
        op = Operator(self, op_desc, *args, **kwargs)
Y
Yu Yang 已提交
380 381 382
        self.ops.appendleft(op)
        return op

Q
Qiao Longfei 已提交
383 384 385 386 387 388 389
    def sync_with_cpp(self):
        # sync variables from cpp
        for var in self.desc.all_vars():
            if not self.has_var(var.name()):
                self.create_var(name=var.name(), desc=var, type=var.type())

        # sync operators from cpp
390 391 392 393
        ops_in_cpp = []
        for op_idx in range(0, self.desc.op_size()):
            ops_in_cpp.append(self.desc.op(op_idx))

Y
Yu Yang 已提交
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409
        if len(self.ops) != 0:
            first_op_in_python = self.ops[0].desc
            last_op_in_python = self.ops[len(self.ops) - 1].desc
            start_index = None
            end_index = None
            for index in range(len(ops_in_cpp)):
                if first_op_in_python == ops_in_cpp[index]:
                    start_index = index
                if last_op_in_python == ops_in_cpp[index]:
                    end_index = index
            assert start_index is not None
            assert end_index is not None
            assert start_index <= end_index
        else:
            start_index = 0
            end_index = -1
Q
Qiao Longfei 已提交
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426

        # sync ops append to the head of cpp_ops
        for index in range((start_index - 1 - 1), -1, -1):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
            self.ops.appendleft(op)

        # sync ops append to the end of cpp_ops
        for index in range((end_index + 1), len(ops_in_cpp)):
            op_desc = ops_in_cpp[index]
            op = Operator(self, op_desc)
            self.ops.append(op)

        assert len(self.ops) == len(ops_in_cpp)
        for index in range(len(self.ops)):
            assert self.ops[index].desc == ops_in_cpp[index]

Q
QI JUN 已提交
427 428 429 430 431 432 433 434 435 436 437
    def _prepend_initialize_ops_(self, param, init_attr):
        op_type = init_attr['type']
        init_attr['shape'] = param.shape
        init_attr['data_type'] = int(param.data_type)
        op = self.prepend_op(
            type=op_type,
            inputs=None,
            outputs={'Out': [param]},
            attrs=init_attr)
        param.op = op

Y
Yu Yang 已提交
438 439

class Program(object):
440 441
    def __init__(self):
        self.desc = core.ProgramDesc()
Y
Yu Yang 已提交
442 443 444
        self.blocks = [Block(self, 0)]
        self.current_block_idx = 0

445 446 447 448 449
    def __str__(self):
        protostr = self.desc.serialize_to_string()
        proto = framework_pb2.ProgramDesc.FromString(str(protostr))
        return proto.__str__()

Y
Yu Yang 已提交
450 451 452 453 454 455
    def clone(self):
        p = Program()
        p.desc = core.ProgramDesc(self.desc)
        p.blocks = [Block(p, i) for i in xrange(self.desc.num_blocks())]
        p.sync_with_cpp()
        return p
456

457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
    def prune(self, targets):
        if not isinstance(targets, list):
            targets = [targets]
        targets_idx = []
        for t in targets:
            if not isinstance(t, Operator):
                if isinstance(t, Variable):
                    t = t.op
                else:
                    raise ValueError(
                        "All targets of prune() can only be Variable or Operator."
                    )

            targets_idx.append([t.block.idx, t.idx])
        res = Program()
        res.desc = core.prune(self.desc, targets_idx)
        res.blocks = [Block(res, i) for i in xrange(res.desc.num_blocks())]
        res.sync_with_cpp()
        return res

477 478 479 480
    @staticmethod
    def parse_from_string(binary_str):
        p = Program()
        p.desc = core.ProgramDesc(binary_str)
481
        p.blocks = [Block(p, i) for i in xrange(p.desc.num_blocks())]
482 483
        p.sync_with_cpp()
        return p
Y
Yu Yang 已提交
484 485 486

    def __repr__(self):
        return str(self)
487

Y
Yu Yang 已提交
488 489 490
    def global_block(self):
        return self.blocks[0]

Q
Qiao Longfei 已提交
491 492 493
    def block(self, index):
        return self.blocks[index]

Y
Yu Yang 已提交
494 495 496
    def current_block(self):
        return self.blocks[self.current_block_idx]

F
fengjiayi 已提交
497
    def append_backward(self, target, no_grad_set=None):
Q
Qiao Longfei 已提交
498 499 500
        """
        return map(param_name -> (grad_name, block_index, op_index))
        """
Q
Qiao Longfei 已提交
501
        assert isinstance(target, Variable)
F
fengjiayi 已提交
502 503
        if no_grad_set is None:
            no_grad_set = set()
Q
Qiao Longfei 已提交
504 505 506 507
        param_to_grad_info = self.desc.append_backward(target.desc, no_grad_set)
        self.sync_with_cpp()
        return param_to_grad_info

Y
Yu Yang 已提交
508 509
    def create_block(self):
        new_block_idx = len(self.blocks)
Y
Yu Yang 已提交
510
        self.desc.append_block(self.current_block().desc)
Y
Yu Yang 已提交
511 512 513 514 515 516 517
        self.current_block_idx = new_block_idx
        self.blocks.append(Block(self, self.current_block_idx))
        return self.current_block()

    def rollback(self):
        self.current_block_idx = self.current_block().parent_idx

Q
Qiao Longfei 已提交
518 519 520 521 522 523
    def sync_with_cpp(self):
        for block_idx in range(len(self.blocks), self.desc.num_blocks()):
            self.blocks.append(Block(self, block_idx))
        for block in self.blocks:
            block.sync_with_cpp()

524 525 526 527 528
    def list_vars(self):
        for each_block in self.blocks:
            for each_var in each_block.vars.itervalues():
                yield each_var

Y
Yu Yang 已提交
529

Y
Yu Yang 已提交
530 531 532 533 534 535 536 537 538 539 540
class Parameter(Variable):
    def __init__(self, block, shape, dtype, **kwargs):
        if shape is None or dtype is None:
            raise ValueError("Parameter must set shape and dtype")
        if len(shape) == 0:
            raise ValueError("Parameter shape cannot be empty")

        for each in shape:
            if each < 0:
                raise ValueError("Parameter shape should not be related with "
                                 "batch-size")
541 542 543

        Variable.__init__(
            self, block, persistable=True, shape=shape, dtype=dtype, **kwargs)
Y
Yu Yang 已提交
544 545 546 547
        self.trainable = kwargs.get('trainable', True)

        self.optimize_attr = kwargs.get('optimize_attr', {'learning_rate': 1.0})

548 549
        self.regularizer = kwargs.get('regularizer', None)

Y
Yu Yang 已提交
550

Y
Yu Yang 已提交
551
# program is a global instance.
Q
Qiao Longfei 已提交
552
g_program = Program()
Q
QI JUN 已提交
553
g_init_program = Program()