提交 97c4d23f 编写于 作者: X xuwei06

Add docs and clean up unused code

上级 0cb8a666
...@@ -3371,7 +3371,7 @@ def make_importer(config_dir, config_args): ...@@ -3371,7 +3371,7 @@ def make_importer(config_dir, config_args):
return Import return Import
default_settings = dict( DEFAULT_SETTING = dict(
batch_size=None, batch_size=None,
mini_batch_size=None, mini_batch_size=None,
algorithm='async_sgd', algorithm='async_sgd',
...@@ -3404,7 +3404,7 @@ default_settings = dict( ...@@ -3404,7 +3404,7 @@ default_settings = dict(
adam_beta2=0.999, adam_beta2=0.999,
adam_epsilon=1e-8, ) adam_epsilon=1e-8, )
settings = copy.deepcopy(default_settings) settings = copy.deepcopy(DEFAULT_SETTING)
settings_deprecated = dict(usage_ratio=1., ) settings_deprecated = dict(usage_ratio=1., )
......
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
import copy import copy
import paddle.trainer.config_parser as config_parser import paddle.trainer.config_parser as config_parser
from paddle.proto.TrainerConfig_pb2 import OptimizationConfig from paddle.proto.TrainerConfig_pb2 import OptimizationConfig
''' '''
This file is a wrapper of formal config_parser. The main idea of this file is to This file is a wrapper of formal config_parser. The main idea of this file is to
separete different config logic into different function, such as network configuration separete different config logic into different function, such as network configuration
...@@ -38,7 +37,7 @@ def parse_network_config(network_conf, config_arg_str=''): ...@@ -38,7 +37,7 @@ def parse_network_config(network_conf, config_arg_str=''):
def parse_optimizer_config(optimizer_conf, config_arg_str=''): def parse_optimizer_config(optimizer_conf, config_arg_str=''):
config_parser.settings = copy.deepcopy(config_parser.default_settings) config_parser.settings = copy.deepcopy(config_parser.DEFAULT_SETTING)
optimizer_conf() optimizer_conf()
opt_config = OptimizationConfig() opt_config = OptimizationConfig()
for k, v in config_parser.settings.iteritems(): for k, v in config_parser.settings.iteritems():
......
...@@ -14,206 +14,55 @@ ...@@ -14,206 +14,55 @@
import collections import collections
import re import re
from paddle.trainer_config_helpers.default_decorators import wrap_name_default
import paddle.trainer_config_helpers as conf_helps import paddle.trainer_config_helpers as conf_helps
from topology import Topology
__layer_map__ = {}
class LayerType(type):
def __new__(cls, name, bases, attrs): def __map_docstr__(doc, name):
method_name = attrs.get('METHOD_NAME', None) if doc is None:
if method_name is not None:
method = getattr(conf_helps, method_name)
if method.__doc__ is not None:
mapper = attrs.get("__map_docstr__", None)
if mapper is not None:
attrs['__doc__'] = LayerType.__map_docstr__(
mapper(method.__doc__),
method_name=method_name,
name=name)
else:
attrs['__doc__'] = LayerType.__map_docstr__(
method.__doc__, method_name=method_name, name=name)
return super(LayerType, cls).__new__(cls, name, bases, attrs)
@staticmethod
def __map_docstr__(doc, name, method_name):
assert isinstance(doc, basestring)
# replace LayerOutput to paddle.v2.config_base.Layer
doc = doc.replace("LayerOutput", "paddle.v2.config_base.Layer")
doc = doc.replace('ParameterAttribute',
'paddle.v2.attr.ParameterAttribute')
doc = re.sub(r'ExtraLayerAttribute[^\s]?',
'paddle.v2.attr.ExtraAttribute', doc)
# xxx_layer to xxx
doc = re.sub(r"(?P<name>[a-z]+)_layer", r"\g<name>", doc)
# XxxxActivation to paddle.v2.Activation.Xxxx
doc = re.sub(r"(?P<name>[A-Z][a-zA-Z]+)Activation",
r"paddle.v2.Activation.\g<name>", doc)
# TODO(yuyang18): Add more rules if needed.
return doc return doc
assert isinstance(doc, basestring)
# replace LayerOutput to paddle.v2.config_base.Layer
doc = doc.replace("LayerOutput", "paddle.v2.config_base.Layer")
doc = doc.replace('ParameterAttribute', 'paddle.v2.attr.ParameterAttribute')
doc = re.sub(r'ExtraLayerAttribute[^\s]?', 'paddle.v2.attr.ExtraAttribute',
doc)
# xxx_layer to xxx
doc = re.sub(r"(?P<name>[a-z]+)_layer", r"\g<name>", doc)
# XxxxActivation to paddle.v2.Activation.Xxxx
doc = re.sub(r"(?P<name>[A-Z][a-zA-Z]+)Activation",
r"paddle.v2.Activation.\g<name>", doc)
# xxx_evaluator to paddle.v2.evaluator.xxx
doc = re.sub(r"(?P<name>[a-z]+)_evaluator", r"evaluator.\g<name>", doc)
# TODO(yuyang18): Add more rules if needed.
return doc
def __convert_to_v2__(f, name, module):
def wrapped(*args, **xargs):
out = f(*args, **xargs)
outs = out
if not isinstance(out, collections.Sequence):
outs = [out]
for l in outs:
if isinstance(l, conf_helps.LayerOutput):
__layer_map__[l.full_name] = l
return out
wrapped.__doc__ = __map_docstr__(f.__doc__, name)
wrapped.__name__ = name
wrapped.__module__ = module
return wrapped
class Layer(object): Layer = conf_helps.LayerOutput
__metaclass__ = LayerType
def __init__(self, name=None, parent_layers=None):
assert isinstance(parent_layers, dict)
self.name = name
self.__context__ = {}
self.__parent_layers__ = parent_layers
# some layer may have some extra parent layer
self.__extra_parent__ = []
# used for evaluator.
self.__children_layers__ = []
def extra_parent(self):
return self.__extra_parent__
def append_extra_parent(self, parent):
self.__extra_parent__.append(parent)
def append_child(self, layer, parent_names):
self.__children_layers__.append((layer, parent_names))
def to_proto(self, context):
"""
function to set proto attribute
"""
self.__context__ = context
# STEP: short cut if this layer is parsed before.
if self.context_name() in context:
if self.use_context_name():
return context[self.context_name()]
else:
return context[self.name]
# STEP: parse extra_parent that is not used by this layer but must
# be parsed before this layer.
for p in self.__extra_parent__:
p.to_proto(context=context)
# STEP: parse parent that is used by this layer, get the result and
# insert into kwargs of the next layer's to_proto_impl method.
kwargs = dict()
for layer_name in self.__parent_layers__:
if not isinstance(self.__parent_layers__[layer_name],
collections.Sequence):
v1_layer = self.__parent_layers__[layer_name].to_proto(
context=context)
else:
v1_layer = map(lambda x: x.to_proto(context=context),
self.__parent_layers__[layer_name])
kwargs[layer_name] = v1_layer
# STEP: parse myself and add myself into context.
ret_val = self.to_proto_impl(**kwargs)
if self.context_name() is not None \
and self.context_name() not in context:
context[self.context_name()] = ret_val
# STEP: parse children that should be pased after this layer.
for layer, pnames in self.__children_layers__:
drop = False
# child will only be parsed if all parents are in context.
for pname in pnames:
if pname not in context:
drop = True
break
if drop:
continue
layer.to_proto(context=context)
# STEP: return v1 layer result
if self.context_name() is None:
return ret_val
elif self.use_context_name():
return context[self.context_name()]
else:
return context[self.name]
def to_proto_impl(self, **kwargs):
raise NotImplementedError()
def context_name(self):
"""
Context name means the context which stores `to_proto_impl` result.
If multiple layer share same context_name, the `to_proto_impl` of them
will be invoked only once.
"""
return self.name
def use_context_name(self):
return False
def calculate_size(self):
"""
lazy calculate size of the layer, should be called when to_proto_impl of
this layer is called.
:return:
"""
return self.__context__[self.context_name()].size
def attr(self):
topo = Topology(self)
return topo.get_layer_proto(self.name)
def __convert_to_v2__(method_name,
parent_names,
is_default_name=True,
attach_parent=False):
if is_default_name:
wrapper = wrap_name_default(name_prefix=method_name)
else:
wrapper = None
class V2LayerImpl(Layer):
METHOD_NAME = method_name
def __init__(self, **kwargs):
parent_layers = dict()
other_kwargs = dict()
for pname in parent_names:
if pname in kwargs:
parent_layers[pname] = kwargs[pname]
if attach_parent:
pnames = [x.context_name() for x in parent_layers.values()]
for pname in parent_layers:
layers = kwargs[pname]
if not isinstance(layers, collections.Sequence):
layers = [layers]
for layer in layers:
layer.append_child(self, pnames)
for key in kwargs.keys():
if key not in parent_names:
other_kwargs[key] = kwargs[key]
name = kwargs.get('name', None)
super(V2LayerImpl, self).__init__(name, parent_layers)
self.__other_kwargs__ = other_kwargs
if wrapper is not None:
__init__ = wrapper(__init__)
def to_proto_impl(self, **kwargs):
args = dict()
for each in kwargs:
args[each] = kwargs[each]
for each in self.__other_kwargs__:
args[each] = self.__other_kwargs__[each]
return getattr(conf_helps, method_name)(**args)
return V2LayerImpl
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
# limitations under the License. # limitations under the License.
import paddle.trainer_config_helpers.evaluators as evs import paddle.trainer_config_helpers.evaluators as evs
import inspect
from config_base import __convert_to_v2__ from config_base import __convert_to_v2__
import inspect
__all__ = [] __all__ = []
...@@ -27,7 +27,8 @@ def initialize(): ...@@ -27,7 +27,8 @@ def initialize():
__ev__ = getattr(evs, __ev_name__) __ev__ = getattr(evs, __ev_name__)
__new_name__ = convert_to_new_name(__ev_name__) __new_name__ = convert_to_new_name(__ev_name__)
globals()[__new_name__] = __ev__ globals()[__new_name__] = __convert_to_v2__(__ev__, __new_name__,
__name__)
globals()[__new_name__].__name__ = __new_name__ globals()[__new_name__].__name__ = __new_name__
__all__.append(__new_name__) __all__.append(__new_name__)
......
...@@ -12,9 +12,9 @@ class Inference(object): ...@@ -12,9 +12,9 @@ class Inference(object):
""" """
Inference combines neural network output and parameters together Inference combines neural network output and parameters together
to do inference. to do inference.
.. code-block:: python .. code-block:: python
inferer = Inference(output_layer=prediction, parameters=parameters) inferer = Inference(output_layer=prediction, parameters=parameters)
for data_batch in batches: for data_batch in batches:
print inferer.infer(data_batch) print inferer.infer(data_batch)
...@@ -92,8 +92,8 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): ...@@ -92,8 +92,8 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
.. code-block:: python .. code-block:: python
result = paddle.infer(output_layer=prediction, result = paddle.infer(output_layer=prediction,
parameters=parameters, parameters=parameters,
input=SomeData) input=SomeData)
print result print result
...@@ -101,14 +101,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): ...@@ -101,14 +101,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
.. code-block:: python .. code-block:: python
result = paddle.infer(output_layer=[prediction1, prediction2], result = paddle.infer(output_layer=[prediction1, prediction2],
parameters=parameters, parameters=parameters,
input=SomeData, input=SomeData,
field=[id, value]]) field=[id, value]])
print result print result
:param output_layer: output of the neural network that would be inferred :param output_layer: output of the neural network that would be inferred
:type output_layer: paddle.v2.config_base.Layer or a list of :type output_layer: paddle.v2.config_base.Layer or a list of
paddle.v2.config_base.Layer paddle.v2.config_base.Layer
:param parameters: parameters of the neural network. :param parameters: parameters of the neural network.
:type parameters: paddle.v2.parameters.Parameters :type parameters: paddle.v2.parameters.Parameters
...@@ -117,14 +117,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'): ...@@ -117,14 +117,14 @@ def infer(output_layer, parameters, input, feeding=None, field='value'):
:type input: collections.Iterable :type input: collections.Iterable
:param feeding: Reader dictionary. Default could generate from input :param feeding: Reader dictionary. Default could generate from input
value. value.
:param field: The prediction field. It should in [`value`, `id`, `prob`]. :param field: The prediction field. It should in [`value`, `id`, `prob`].
`value` and `prob` mean return the prediction probabilities, `value` and `prob` mean return the prediction probabilities,
`id` means return the prediction labels. Default is `value`. `id` means return the prediction labels. Default is `value`.
Note that `prob` only used when output_layer is beam_search Note that `prob` only used when output_layer is beam_search
or max_id. or max_id.
:type field: str :type field: str
:return: The prediction result. If there are multiple outout_layers and fields, :return: The prediction result. If there are multiple outout_layers and fields,
the return order is outout_layer1.field1, outout_layer2.field1, ..., the return order is outout_layer1.field1, outout_layer2.field1, ...,
outout_layer1.field2, outout_layer2.field2 ... outout_layer1.field2, outout_layer2.field2 ...
:rtype: numpy.ndarray :rtype: numpy.ndarray
""" """
......
...@@ -33,26 +33,14 @@ The primary usage shows below. ...@@ -33,26 +33,14 @@ The primary usage shows below.
import collections import collections
import copy import copy
import re
import paddle.trainer_config_helpers.layers as v1_layers import paddle.trainer_config_helpers.layers as v1_layers
import paddle.trainer.config_parser as cp import paddle.trainer.config_parser as cp
from paddle.proto.ModelConfig_pb2 import ModelConfig, SubModelConfig from paddle.proto.ModelConfig_pb2 import ModelConfig, SubModelConfig
from config_base import __convert_to_v2__
import config_base
__all__ = ['data', 'parse_network'] __all__ = ['data', 'parse_network']
__layer_map__ = {}
def __wrap__(f):
def wrapped(*args, **xargs):
out = f(*args, **xargs)
outs = out
if not isinstance(out, collections.Sequence):
outs = [out]
for l in outs:
if isinstance(l, v1_layers.LayerOutput):
__layer_map__[l.full_name] = l
return out
return wrapped
def __need_to_keep__(name): def __need_to_keep__(name):
...@@ -90,7 +78,7 @@ for name in v1_layers.__all__: ...@@ -90,7 +78,7 @@ for name in v1_layers.__all__:
continue continue
new_name = __convert_name__(name) new_name = __convert_name__(name)
if callable(obj) and __need_to_wrap__(name): if callable(obj) and __need_to_wrap__(name):
globals()[new_name] = __wrap__(obj) globals()[new_name] = __convert_to_v2__(obj, new_name, __name__)
else: else:
globals()[new_name] = obj globals()[new_name] = obj
__all__.append(new_name) __all__.append(new_name)
...@@ -102,9 +90,21 @@ def __data_layer__(name, type, **kwargs): ...@@ -102,9 +90,21 @@ def __data_layer__(name, type, **kwargs):
return l return l
data = __wrap__(__data_layer__) def __map_data_docstr__(doc):
doc = re.sub(r'(data = [^\)]+)\).*',
"data = paddle.layer.data(name=\"input\", "
"type=paddle.data_type.dense_vector(1000))", doc)
doc = re.sub(r':param size:.*', ':param type: Data type of this data layer',
doc)
doc = re.sub(r':type size:.*', ":type size: paddle.v2.data_type.InputType",
doc)
return doc
__data_layer__.__doc__ = __map_data_docstr__(v1_layers.data_layer.__doc__)
LayerV2 = v1_layers.LayerOutput data = __convert_to_v2__(__data_layer__, 'name', __name__)
def __get_used_layers__(output_layers, extra_layers=None): def __get_used_layers__(output_layers, extra_layers=None):
...@@ -273,7 +273,7 @@ def parse_network(output_layers, extra_layers=None): ...@@ -273,7 +273,7 @@ def parse_network(output_layers, extra_layers=None):
def get_layer(name): def get_layer(name):
return __layer_map__.get(name) return config_base.__layer_map__.get(name)
cp.begin_parse() cp.begin_parse()
...@@ -32,6 +32,7 @@ class RNNTest(unittest.TestCase): ...@@ -32,6 +32,7 @@ class RNNTest(unittest.TestCase):
def parse_old_rnn(): def parse_old_rnn():
reset_parser() reset_parser()
def step(y): def step(y):
mem = conf_helps.memory(name="rnn_state", size=hidden_dim) mem = conf_helps.memory(name="rnn_state", size=hidden_dim)
out = conf_helps.fc_layer( out = conf_helps.fc_layer(
...@@ -52,6 +53,7 @@ class RNNTest(unittest.TestCase): ...@@ -52,6 +53,7 @@ class RNNTest(unittest.TestCase):
def parse_new_rnn(): def parse_new_rnn():
reset_parser() reset_parser()
def new_step(y): def new_step(y):
mem = layer.memory(name="rnn_state", size=hidden_dim) mem = layer.memory(name="rnn_state", size=hidden_dim)
out = layer.fc(input=[y, mem], out = layer.fc(input=[y, mem],
...@@ -72,7 +74,6 @@ class RNNTest(unittest.TestCase): ...@@ -72,7 +74,6 @@ class RNNTest(unittest.TestCase):
parse_new_rnn().splitlines(1)) parse_new_rnn().splitlines(1))
print ''.join(diff) print ''.join(diff)
def test_sequence_rnn_multi_input(self): def test_sequence_rnn_multi_input(self):
dict_dim = 10 dict_dim = 10
word_dim = 8 word_dim = 8
...@@ -81,6 +82,7 @@ class RNNTest(unittest.TestCase): ...@@ -81,6 +82,7 @@ class RNNTest(unittest.TestCase):
def parse_old_rnn(): def parse_old_rnn():
reset_parser() reset_parser()
def test(): def test():
data = conf_helps.data_layer(name="word", size=dict_dim) data = conf_helps.data_layer(name="word", size=dict_dim)
label = conf_helps.data_layer(name="label", size=label_dim) label = conf_helps.data_layer(name="label", size=label_dim)
......
...@@ -17,34 +17,11 @@ import collections ...@@ -17,34 +17,11 @@ import collections
from paddle.proto.ModelConfig_pb2 import ModelConfig from paddle.proto.ModelConfig_pb2 import ModelConfig
import paddle.trainer_config_helpers as conf_helps import paddle.trainer_config_helpers as conf_helps
import layer as v2_layer import layer as v2_layer
import config_base
__all__ = ['Topology'] __all__ = ['Topology']
def __flatten__(lis):
"""
Given a list, possibly nested to any level, return it flattened.
"""
new_lis = []
for item in lis:
if isinstance(item, collections.Sequence):
new_lis.extend(__flatten__(item))
else:
new_lis.append(item)
return new_lis
def __bfs_travel__(callback, *layers):
layers = __flatten__(layers)
for each_layer in layers:
__break__ = callback(each_layer)
if __break__:
return
__layers__ = each_layer.__parent_layers__.values() + \
each_layer.extra_parent()
__bfs_travel__(callback, *__layers__)
class Topology(object): class Topology(object):
""" """
Topology is used to store the information about all layers Topology is used to store the information about all layers
...@@ -125,5 +102,5 @@ class Topology(object): ...@@ -125,5 +102,5 @@ class Topology(object):
def __check_layer_type__(layer): def __check_layer_type__(layer):
if not isinstance(layer, v2_layer.LayerV2): if not isinstance(layer, config_base.Layer):
raise ValueError('layer should have type paddle.layer.Layer') raise ValueError('layer should have type paddle.v2.config_base.Layer')
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册