From a1ba3f442fd80382969ed2c434a66985be1e2c1f Mon Sep 17 00:00:00 2001 From: qijun Date: Sat, 12 Nov 2016 02:26:18 +0000 Subject: [PATCH] format python code in python directory --- python/paddle/__init__.py | 1 - python/paddle/trainer/PyDataProvider2.py | 35 +- .../paddle/trainer/PyDataProviderWrapper.py | 35 +- python/paddle/trainer/__init__.py | 1 - python/paddle/trainer/config_parser.py | 2008 ++++++++--------- .../paddle/trainer/config_parser_extension.py | 10 +- python/paddle/trainer/recurrent_units.py | 489 ++-- .../trainer_config_helpers/activations.py | 51 +- python/paddle/trainer_config_helpers/attrs.py | 37 +- .../trainer_config_helpers/data_sources.py | 55 +- .../default_decorators.py | 19 +- .../trainer_config_helpers/evaluators.py | 241 +- .../paddle/trainer_config_helpers/layers.py | 1433 +++++++----- python/paddle/trainer_config_helpers/math.py | 27 +- .../paddle/trainer_config_helpers/networks.py | 772 ++++--- .../trainer_config_helpers/optimizers.py | 51 +- .../paddle/trainer_config_helpers/poolings.py | 23 +- .../tests/configs/img_layers.py | 19 +- .../tests/configs/img_trans_layers.py | 20 +- .../tests/configs/last_first_seq.py | 17 +- .../tests/configs/layer_activations.py | 16 +- .../tests/configs/math_ops.py | 8 +- .../tests/configs/projections.py | 29 +- .../tests/configs/shared_fc.py | 27 +- .../tests/configs/shared_lstm.py | 28 +- .../tests/configs/simple_rnn_layers.py | 31 +- .../tests/configs/test_bi_grumemory.py | 5 +- .../tests/configs/test_bilinear_interp.py | 37 +- .../tests/configs/test_cost_layers.py | 49 +- .../configs/test_cost_layers_with_weight.py | 12 +- .../tests/configs/test_expand_layer.py | 14 +- .../tests/configs/test_fc.py | 12 +- .../tests/configs/test_grumemory_layer.py | 14 +- .../tests/configs/test_hsigmoid.py | 7 +- .../tests/configs/test_lstmemory_layer.py | 14 +- .../tests/configs/test_maxout.py | 68 +- .../tests/configs/test_ntm_layers.py | 35 +- .../tests/configs/test_print_layer.py | 5 +- .../tests/configs/test_rnn_group.py | 25 +- .../tests/configs/test_sequence_pooling.py | 21 +- .../tests/configs/test_split_datasource.py | 14 +- .../tests/configs/test_spp_layer.py | 16 +- .../tests/configs/unused_layers.py | 7 +- .../tests/configs/util_layers.py | 8 +- .../tests/layers_test_config.py | 65 +- python/paddle/trainer_config_helpers/utils.py | 4 +- python/paddle/utils/image_util.py | 76 +- python/paddle/utils/make_model_diagram.py | 17 +- python/paddle/utils/plotcurve.py | 40 +- python/paddle/utils/predefined_net.py | 240 +- python/paddle/utils/preprocess_img.py | 37 +- python/paddle/utils/preprocess_util.py | 65 +- python/paddle/utils/show_pb.py | 9 +- python/paddle/utils/torch2paddle.py | 25 +- 54 files changed, 3498 insertions(+), 2926 deletions(-) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 7f9e87eee6..c90af2ee00 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index 53409b746d..0c577ec657 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -18,9 +18,8 @@ import collections import functools import itertools -logging.basicConfig( - format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" - " %(message)s") +logging.basicConfig(format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" + " %(message)s") class SequenceType(object): @@ -132,8 +131,10 @@ class InputOrderWrapper(object): def __call__(self, obj, filename): for item in self.generator(obj, filename): if isinstance(item, dict): - yield [item.get(input_name, None) for input_name in - self.input_order] + yield [ + item.get(input_name, None) + for input_name in self.input_order + ] else: yield item @@ -162,8 +163,8 @@ class CheckWrapper(object): yield items except AssertionError as e: self.logger.warning( - "Item (%s) is not fit the input type with error %s" - % (repr(item), repr(e))) + "Item (%s) is not fit the input type with error %s" % + (repr(item), repr(e))) if self.check_fail_continue: continue @@ -202,13 +203,17 @@ class CheckWrapper(object): callback(each) -def provider(input_types=None, should_shuffle=None, pool_size=-1, +def provider(input_types=None, + should_shuffle=None, + pool_size=-1, min_pool_size=-1, can_over_batch_size=True, calc_batch_size=None, cache=CacheType.NO_CACHE, - check=False, check_fail_continue=False, - init_hook=None, **kwargs): + check=False, + check_fail_continue=False, + init_hook=None, + **kwargs): """ Provider decorator. Use it to make a function into PyDataProvider2 object. In this function, user only need to get each sample for some train/test @@ -318,9 +323,9 @@ def provider(input_types=None, should_shuffle=None, pool_size=-1, "Could not recognize should_shuffle (%s), " "just use default value of should_shuffle." " Please set should_shuffle to bool value or " - "something in %s" % ( - repr(self.should_shuffle), - repr(true_table + false_table))) + "something in %s" % + (repr(self.should_shuffle), + repr(true_table + false_table))) self.should_shuffle = None self.pool_size = pool_size @@ -351,8 +356,7 @@ def provider(input_types=None, should_shuffle=None, pool_size=-1, self.generator = InputOrderWrapper(self.generator, self.input_order) if self.check: - self.generator = CheckWrapper(self.generator, - self.slots, + self.generator = CheckWrapper(self.generator, self.slots, check_fail_continue, self.logger) @@ -368,4 +372,3 @@ def deserialize_args(args): :return: """ return cPickle.loads(args) - diff --git a/python/paddle/trainer/PyDataProviderWrapper.py b/python/paddle/trainer/PyDataProviderWrapper.py index c4b907af54..90b684a000 100644 --- a/python/paddle/trainer/PyDataProviderWrapper.py +++ b/python/paddle/trainer/PyDataProviderWrapper.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ This module provide a wrapper(decorator) to wrap a data process method into a PyDataProvider. Some examples are shown `here `_. @@ -47,6 +46,7 @@ except ImportError: import io + class SlotType(object): # Just a hint for user. pass @@ -83,6 +83,7 @@ class SparseNonValueSlot(SlotType): - **SubSeq**: [[[int, int, ...], [int, ....], ...] , \ [[int, int, ...], [int, ....], ...] , ...] """ + def __init__(self, dim): """ :param dim: slot dimension @@ -294,8 +295,9 @@ class GeneralPyDataProvider: fn = "%s_%d" % (self.profile_filename, self.profile_count) sortby = "cumulative" with open(fn, "w") as f: - pstats.Stats(self.profiler, stream=f).sort_stats( - sortby).print_stats() + pstats.Stats( + self.profiler, + stream=f).sort_stats(sortby).print_stats() self.logger.info("saving profile to file %s" % fn) self.profile_count += 1 self.logger.info("resetting profile") @@ -453,9 +455,10 @@ class GeneralPyDataProvider: seq_stream.flush() subseq_stream.flush() - return "".join([self.int_packer.pack(current_batch_size), - data_bytes.getvalue(), - seq_bytes.getvalue(), subseq_bytes.getvalue()]) + return "".join([ + self.int_packer.pack(current_batch_size), data_bytes.getvalue(), + seq_bytes.getvalue(), subseq_bytes.getvalue() + ]) finally: data_stream.close() @@ -516,7 +519,7 @@ class GeneralPyDataProvider: self.data_pool[idx]) idx -= 1 - ret_list += self.data_pool[self.data_pool_idx: idx + 1] + ret_list += self.data_pool[self.data_pool_idx:idx + 1] # for speed reason, just shift left index, not delete data actually. self.data_pool_idx = idx + 1 @@ -537,8 +540,8 @@ class GeneralPyDataProvider: if self.max_pool_size == 0: for i in xrange(min(self.file_count, len(self.generators))): self.data_pool += list(self.generators[i]) - self.generators = self.generators[ - min(self.file_count, len(self.generators)):] + self.generators = self.generators[min(self.file_count, + len(self.generators)):] self.max_pool_size = len(self.data_pool) else: while len(self.data_pool) < self.max_pool_size and len( @@ -562,9 +565,15 @@ def default_init_hook(cls, *args, **kwargs): del cls, args, kwargs -def provider(slots=None, use_seq=False, should_shuffle=True, pool_size=1, - can_over_batch_size=True, calc_batch_size=lambda data: 1, - debug=False, init_hook=default_init_hook, profile_filename=None): +def provider(slots=None, + use_seq=False, + should_shuffle=True, + pool_size=1, + can_over_batch_size=True, + calc_batch_size=lambda data: 1, + debug=False, + init_hook=default_init_hook, + profile_filename=None): """ The decorator for PyDataProvider. User should use this to create Provider class. User should only concern how to read sample from file. @@ -663,7 +672,7 @@ def provider(slots=None, use_seq=False, should_shuffle=True, pool_size=1, def __init__(self, *file_list, **kwargs): logging.basicConfig( format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" - " %(message)s") + " %(message)s") self.logger = logging.getLogger("") if debug: diff --git a/python/paddle/trainer/__init__.py b/python/paddle/trainer/__init__.py index 7f9e87eee6..c90af2ee00 100644 --- a/python/paddle/trainer/__init__.py +++ b/python/paddle/trainer/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index eec978e1fa..881f0b8214 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -13,7 +13,6 @@ # limitations under the License. from __future__ import print_function - ''' The following functions are available in the config file: @@ -101,50 +100,45 @@ except Exception as e: raise logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) __real_print__ = print -print=logger.info +print = logger.info # from layer type name to layer class g_layer_type_map = {} + # Initialize global variables. We use this function so that we can # call parse_config() multiple times def init_config_environment( - g_default_momentum = None, - g_default_decay_rate = None, - g_default_initial_mean = 0., - g_default_initial_std = 0.01, - g_default_num_batches_regularization = None, - g_default_initial_strategy = 0, - g_default_initial_smart = False, - g_default_gradient_clipping_threshold = None, - g_default_device = None, - g_default_update_hooks = None, - g_default_compact_func = None, - - g_config = TrainerConfig(), - g_layer_map = {}, - g_parameter_map = {}, - - g_extended_config_funcs = {}, + g_default_momentum=None, + g_default_decay_rate=None, + g_default_initial_mean=0., + g_default_initial_std=0.01, + g_default_num_batches_regularization=None, + g_default_initial_strategy=0, + g_default_initial_smart=False, + g_default_gradient_clipping_threshold=None, + g_default_device=None, + g_default_update_hooks=None, + g_default_compact_func=None, + g_config=TrainerConfig(), + g_layer_map={}, + g_parameter_map={}, + g_extended_config_funcs={}, # store command args of paddle_trainer - g_command_config_args = {}, + g_command_config_args={}, # Used for PyDataProvider to avoid duplicate module name - g_py_module_name_list = [], - - g_current_submodel = None, - g_root_submodel = None, - g_submodel_map = {}, - g_submodel_stack = [], - - g_add_submodel_suffix = False, - ): + g_py_module_name_list=[], + g_current_submodel=None, + g_root_submodel=None, + g_submodel_map={}, + g_submodel_stack=[], + g_add_submodel_suffix=False, ): for k, v in locals().iteritems(): globals()[k] = copy.deepcopy(v) @@ -161,43 +155,54 @@ def config_assert(b, msg): if not b: logger.fatal(msg) + g_config_funcs = {} + # decorator for indicating a function which can be used in config file def config_func(func): g_config_funcs[func.func_name] = func return func + # decorator for indicating a class which can be used in config file def config_class(cls): g_config_funcs[cls.__name__] = cls return cls + # decorator for indicating a class for a layer type def config_layer(layer_type): def wrap(cls): g_config_funcs[cls.__name__] = cls g_layer_type_map[layer_type] = cls return cls + return wrap + def gen_parameter_name(layer_name, input_index): return '_%s.w%d' % (layer_name, input_index) + def gen_bias_parameter_name(layer_name): return '_%s.wbias' % layer_name + def default(x, default_value): return default_value if x is None else x + class Cfg(object): def add_keys(self, locals): for k, v in locals.iteritems(): if not k.startswith('_'): self.__setattr__(k, v) + # functions available in config file + # Define the name of the input layers of the NeuralNetwork. # The type of these layers must be "data". # These layers will be provided with the DataBatch obtained @@ -216,6 +221,7 @@ def Inputs(*args): if g_current_submodel is g_root_submodel: g_config.model_config.input_layer_names.append(name) + @config_func def HasInputsSet(): return len(g_current_submodel.input_layer_names) != 0 @@ -244,7 +250,7 @@ def SubModelBegin(name): global g_current_submodel, g_root_submodel, g_submodel_stack g_submodel_stack.append(g_current_submodel) - name = MakeLayerNameInParentSubmodel(name) #rename in nested submodel + name = MakeLayerNameInParentSubmodel(name) #rename in nested submodel config_assert(name not in g_submodel_map, 'Duplicated submodel name: %s' % name) @@ -254,36 +260,42 @@ def SubModelBegin(name): g_submodel_map[name] = sub_model g_current_submodel = sub_model + @config_func -def SubModelEnd(name = None): +def SubModelEnd(name=None): global g_current_submodel, g_root_submodel, g_submodel_stack - config_assert(g_current_submodel is not g_root_submodel, "submodel not begin") + config_assert(g_current_submodel is not g_root_submodel, + "submodel not begin") if name is not None: - config_assert(g_current_submodel.name == MakeLayerNameInParentSubmodel(name), - "submodel name error") + config_assert( + g_current_submodel.name == MakeLayerNameInParentSubmodel(name), + "submodel name error") g_current_submodel = g_submodel_stack.pop() + def MakeLayerNameInParentSubmodel(name): suffix = "" if len(g_submodel_stack) > 1: suffix = "@" + g_submodel_stack[-1].name return name + suffix + def GetLayerBaseName(name): return name.split('@')[0] -def MakeLayerNameInSubmodel(name, submodel_name = None): + +def MakeLayerNameInSubmodel(name, submodel_name=None): global g_current_submodel global g_add_submodel_suffix - if (submodel_name is None - and not g_add_submodel_suffix - and not g_current_submodel.is_recurrent_layer_group): + if (submodel_name is None and not g_add_submodel_suffix and + not g_current_submodel.is_recurrent_layer_group): return name if submodel_name is None: submodel_name = g_current_submodel.name return name + "@" + submodel_name + # Define a recurrent layer group begin with RecurrentLayerGroupBegin # and end with RecurrentLayerGroupEnd. # A recurrent layer group forward/backward one frame after previous frame @@ -332,8 +344,10 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, if in_links_count == 0: in_links_has_subseq = has_subseq else: - config_assert(in_links_has_subseq == has_subseq, - "The sequence type of in_links should be the same in RecurrentLayerGroup") + config_assert( + in_links_has_subseq == has_subseq, + "The sequence type of in_links should be the same in RecurrentLayerGroup" + ) in_links_count += 1 layer_name = MakeLayerNameInParentSubmodel(name) layer = g_layer_map[layer_name] @@ -347,6 +361,7 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, pair.link_name = MakeLayerNameInSubmodel(name) pair.has_subseq = has_subseq + @config_func def RecurrentLayerGroupSetOutLink(link): if isinstance(link, basestring): @@ -363,8 +378,7 @@ def RecurrentLayerGroupSetOutLink(link): def RecurrentLayerGroupSetGenerator(generator=None): - generator.eos_layer_name = MakeLayerNameInSubmodel( - generator.eos_layer_name) + generator.eos_layer_name = MakeLayerNameInSubmodel(generator.eos_layer_name) g_current_submodel.generator.CopyFrom(generator) @@ -375,21 +389,18 @@ def RecurrentLayerGroupBegin(name, generator=None, target_inlinkname="", seq_reversed=False): - RecurrentLayerGroupWithoutOutLinksBegin(name, - in_links, - seq_reversed, + RecurrentLayerGroupWithoutOutLinksBegin(name, in_links, seq_reversed, target_inlinkname) for link in out_links: RecurrentLayerGroupSetOutLink(link) - if generator is not None: RecurrentLayerGroupSetGenerator(generator) - config_assert(len(in_links) == 0, - "no in_links should be passed to generator") - config_assert(len(out_links) >= 1, - "one or more than one out_links should be passed to generator") - + config_assert( + len(in_links) == 0, "no in_links should be passed to generator") + config_assert( + len(out_links) >= 1, + "one or more than one out_links should be passed to generator") @config_func @@ -397,9 +408,10 @@ def RecurrentLayerGroupEnd(name): global g_current_submodel config_assert(g_current_submodel.is_recurrent_layer_group, "RecurrentLayerGroup not begin") - for pair in g_current_submodel.memories: #check exist + for pair in g_current_submodel.memories: #check exist layer = g_layer_map[pair.layer_name] - config_assert(layer is not None, "memory declare wrong name:%s" % pair.layer_name) + config_assert(layer is not None, "memory declare wrong name:%s" % + pair.layer_name) memory_link = g_layer_map[pair.link_name] config_assert(layer.size == memory_link.size, "memory declare wrong size:%d" % memory_link.size) @@ -418,12 +430,14 @@ def RecurrentLayerGroupEnd(name): else: GatherAgentLayer(name=agent_name, size=layer.size) + # Define the model type # currently, the paddle supports "nn", "recurrent_nn", "recursive_nn" and "multi_nn" @config_func def model_type(name): g_config.model_config.type = name + @config_class class Bias(Cfg): def __init__( @@ -441,10 +455,10 @@ class Bias(Cfg): sparse_remote_update=None, gradient_clipping_threshold=None, is_static=None, - is_shared=None, - ): + is_shared=None, ): self.add_keys(locals()) + # Define one input for a layer @config_class class Input(Cfg): @@ -477,19 +491,20 @@ class Input(Cfg): is_static=None, is_shared=None, update_hooks=None, - input_layer_argument=None, - ): + input_layer_argument=None, ): self.add_keys(locals()) self.input_layer_name = MakeLayerNameInSubmodel(input_layer_name) + # Define a projection for iexed layer @config_class class Projection(Input): - type = None # subclass should set it correctly + type = None # subclass should set it correctly + def __init__( self, input_layer_name, - size = 0, # projection output size + size=0, # projection output size parameter_name=None, learning_rate=None, momentum=None, @@ -509,8 +524,7 @@ class Projection(Input): is_static=None, is_shared=None, update_hooks=None, - input_layer_argument=None, - ): + input_layer_argument=None, ): self.add_keys(locals()) self.input_layer_name = MakeLayerNameInSubmodel(input_layer_name) @@ -524,8 +538,10 @@ class Projection(Input): # to indicate using the size from Layer config def calc_output_size(self, input_layer_config): return self.size + def calc_parameter_size(self, input_size, output_size): raise NotimplementedError + def calc_parameter_dims(self, input_size, output_size): raise NotimplementedError @@ -536,31 +552,32 @@ class IdentityProjection(Projection): def calc_output_size(self, input_layer_config): return input_layer_config.size + def calc_parameter_size(self, input_size, output_size): return 0 + def calc_parameter_dims(self, input_size, output_size): return [] + # Like IdentityProjection, but layer size may smaller than input size, # the projection select dimesions [offset, offset+layer_size) from input @config_class class IdentityOffsetProjection(Projection): type = 'identity_offset' - def __init__( - self, - input_layer_name, - offset, - **xargs): - super(IdentityOffsetProjection, self).__init__( - input_layer_name, **xargs) + def __init__(self, input_layer_name, offset, **xargs): + super(IdentityOffsetProjection, self).__init__(input_layer_name, + **xargs) self.proj_conf.offset = offset def calc_parameter_size(self, input_size, output_size): return 0 + def calc_parameter_dims(self, input_size, output_size): return [] + # DotMulProjection performs element-wise multiplication with weight @config_class class DotMulProjection(Projection): @@ -568,49 +585,53 @@ class DotMulProjection(Projection): def calc_output_size(self, input_layer_config): return input_layer_config.size + def calc_parameter_size(self, input_size, output_size): return output_size + def calc_parameter_dims(self, input_size, output_size): return [1, output_size] + @config_class class TableProjection(Projection): type = 'table' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [input_size, output_size] + @config_class class FullMatrixProjection(Projection): type = 'fc' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [input_size, output_size] + @config_class class TransposedFullMatrixProjection(Projection): type = 'trans_fc' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [output_size, input_size] + @config_class class ContextProjection(Projection): type = 'context' - def __init__( - self, - input_layer_name, - context_start, - context_length, - trainable_padding, - **xargs): + def __init__(self, input_layer_name, context_start, context_length, + trainable_padding, **xargs): super(ContextProjection, self).__init__(input_layer_name, **xargs) self.proj_conf.context_start = context_start self.proj_conf.context_length = context_length @@ -638,23 +659,21 @@ class ContextProjection(Projection): class ConvProjection(Projection): type = 'conv' - def __init__( - self, - input_layer_name, - num_filters=None, - conv_conf=None, - **xargs): + def __init__(self, + input_layer_name, + num_filters=None, + conv_conf=None, + **xargs): super(ConvProjection, self).__init__(input_layer_name, **xargs) if num_filters is not None: self.proj_conf.num_filters = num_filters - parse_conv(conv_conf, - input_layer_name, - self.proj_conf.conv_conf, + parse_conv(conv_conf, input_layer_name, self.proj_conf.conv_conf, num_filters) # TODO: support rectangle input - self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x ** 2) * num_filters + self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x** + 2) * num_filters def calc_output_size(self, input_layer_config): return self.proj_conf.output_size @@ -672,14 +691,15 @@ class ConvProjection(Projection): def calc_parameter_dims(self, input_size, output_size): return None + # Define a operator for mixed layer @config_class class Operator(Cfg): - type = None # subclass should set it correctly + type = None # subclass should set it correctly + def __init__( self, - input_layer_names, - ): + input_layer_names, ): self.add_keys(locals()) self.operator_conf = OperatorConfig() self.operator_conf.type = self.type @@ -690,16 +710,13 @@ class Operator(Cfg): def calc_output_size(self, input_sizes): return 0 + @config_class class DotMulOperator(Operator): type = 'dot_mul' - def __init__( - self, - input_layer_names, - scale=None, - **xargs): - super(DotMulOperator, self).__init__( - input_layer_names, **xargs) + + def __init__(self, input_layer_names, scale=None, **xargs): + super(DotMulOperator, self).__init__(input_layer_names, **xargs) if scale is not None: self.operator_conf.dotmul_scale = scale @@ -715,26 +732,24 @@ class DotMulOperator(Operator): return input_sizes[0] - @config_class class ConvOperator(Operator): type = 'conv' - def __init__( - self, - input_layer_names, - num_filters=None, - conv_conf=None, - **xargs): - super(ConvOperator, self).__init__( - input_layer_names, **xargs) + + def __init__(self, + input_layer_names, + num_filters=None, + conv_conf=None, + **xargs): + super(ConvOperator, self).__init__(input_layer_names, **xargs) if num_filters is not None: self.operator_conf.num_filters = num_filters parse_conv(conv_conf, MakeLayerNameInSubmodel(input_layer_names[0]), - self.operator_conf.conv_conf, - num_filters) - self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x ** 2) * num_filters + self.operator_conf.conv_conf, num_filters) + self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x + **2) * num_filters config_assert(len(input_layer_names) == 2, "Conv is binary operator") @@ -745,119 +760,106 @@ class ConvOperator(Operator): # please refer to the comments in proto/ModelConfig.proto @config_class class Conv(Cfg): - def __init__( - self, - filter_size, - channels, - padding = None, - stride = None, - groups = None, - filter_channels = None, - output_x = None, - img_size = None, - caffe_mode = True, - filter_size_y = None, - padding_y = None, - stride_y = None): + def __init__(self, + filter_size, + channels, + padding=None, + stride=None, + groups=None, + filter_channels=None, + output_x=None, + img_size=None, + caffe_mode=True, + filter_size_y=None, + padding_y=None, + stride_y=None): self.add_keys(locals()) if filter_size_y is None: - self.filter_size_y = filter_size + self.filter_size_y = filter_size if padding_y is None: - self.padding_y = padding + self.padding_y = padding if stride_y is None: - self.stride_y = stride + self.stride_y = stride if output_x is not None: - config_assert(output_x <= 0) + config_assert(output_x <= 0) + # please refer to the comments in proto/ModelConfig.proto @config_class class BilinearInterp(Cfg): - def __init__( - self, - out_size_x = None, - out_size_y = None, - num_channels = None): + def __init__(self, out_size_x=None, out_size_y=None, num_channels=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Pool(Cfg): - def __init__( - self, - pool_type, - channels, - size_x, - size_y = None, - img_width = None, - start = None, - stride = None, - stride_y = None, - padding = None, - padding_y = None): + def __init__(self, + pool_type, + channels, + size_x, + size_y=None, + img_width=None, + start=None, + stride=None, + stride_y=None, + padding=None, + padding_y=None): self.add_keys(locals()) - + + # please refer to the comments in proto/ModelConfig.proto @config_class class SpatialPyramidPool(Cfg): - def __init__( - self, - pool_type, - pyramid_height, - channels, - img_width = None): + def __init__(self, pool_type, pyramid_height, channels, img_width=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Norm(Cfg): - def __init__( - self, - norm_type, - channels, - size, - scale, - pow, - output_x = None, - img_size = None, - blocked = None): + def __init__(self, + norm_type, + channels, + size, + scale, + pow, + output_x=None, + img_size=None, + blocked=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Image(Cfg): - def __init__( - self, - channels, - img_size = None): + def __init__(self, channels, img_size=None): self.add_keys(locals()) + @config_class class BlockExpand(Cfg): - def __init__( - self, - channels, - padding_x = 0, - padding_y = 0, - stride_x = 0, - stride_y = 0, - block_x = 0, - block_y = 0, - img_size_x = 0, - img_size_y = 0, - output_x = 0, - output_y = 0): + def __init__(self, + channels, + padding_x=0, + padding_y=0, + stride_x=0, + stride_y=0, + block_x=0, + block_y=0, + img_size_x=0, + img_size_y=0, + output_x=0, + output_y=0): self.add_keys(locals()) + @config_class class MaxOut(Cfg): - def __init__( - self, - channels, - groups, - img_size_x = 0, - img_size_y = 0): + def __init__(self, channels, groups, img_size_x=0, img_size_y=0): self.add_keys(locals()) + def DataBase(async_load_data=False, constant_slots=None, data_ratio=1, @@ -871,23 +873,23 @@ def DataBase(async_load_data=False, if constant_slots: data_config.constant_slots.extend(constant_slots) - data_config.data_ratio=data_ratio - data_config.is_main_data=is_main_data + data_config.data_ratio = data_ratio + data_config.is_main_data = is_main_data - usage_ratio=default(usage_ratio, settings_deprecated["usage_ratio"]) + usage_ratio = default(usage_ratio, settings_deprecated["usage_ratio"]) config_assert(usage_ratio >= 0 and usage_ratio <= 1, "The range of usage_ratio is [0, 1]") data_config.usage_ratio = usage_ratio return data_config + @config_func -def SimpleData( - files=None, - feat_dim=None, - context_len=None, - buffer_capacity=None, - **xargs): +def SimpleData(files=None, + feat_dim=None, + context_len=None, + buffer_capacity=None, + **xargs): data_config = DataBase(**xargs) data_config.type = 'simple' data_config.files = files @@ -898,31 +900,36 @@ def SimpleData( data_config.buffer_capacity = buffer_capacity return data_config + @config_func -def PyData( - files=None, - type=None, - file_group_queue_capacity=None, - load_data_module=None, - load_data_object=None, - load_data_args="", - load_file_count=None, - constant_slots=None, - load_thread_num=None, - **xargs): +def PyData(files=None, + type=None, + file_group_queue_capacity=None, + load_data_module=None, + load_data_object=None, + load_data_args="", + load_file_count=None, + constant_slots=None, + load_thread_num=None, + **xargs): data_config = DataBase(**xargs) data_config.type = 'py' if load_data_module in g_py_module_name_list: + def get_path(module): m = __import__(load_data_module) return os.path.split(os.path.realpath(m.__file__))[0] + # python C-api is not thread safe, one module can only be import once, # so here we nedd to copy the module with different names if it has to be # imported several times. - module_new_name = "%s_copy_%d" % (load_data_module, len(g_py_module_name_list)) + module_new_name = "%s_copy_%d" % (load_data_module, + len(g_py_module_name_list)) g_py_module_name_list.append(module_new_name) - module_path = "%s/%s.py" % (get_path(load_data_module), load_data_module) - new_module_path = "%s/%s.py" % (get_path(load_data_module), module_new_name) + module_path = "%s/%s.py" % (get_path(load_data_module), + load_data_module) + new_module_path = "%s/%s.py" % (get_path(load_data_module), + module_new_name) if os.path.isfile(module_path) == False: raise Exception("File %s is not exist." % module_path) shutil.copy2(module_path, new_module_path) @@ -947,15 +954,15 @@ def PyData( data_config.constant_slots.extend(constant_slots) return data_config + @config_func -def ProtoData( - files=None, - type=None, - file_group_queue_capacity=None, - load_file_count=None, - constant_slots=None, - load_thread_num=None, - **xargs): +def ProtoData(files=None, + type=None, + file_group_queue_capacity=None, + load_file_count=None, + constant_slots=None, + load_thread_num=None, + **xargs): data_config = DataBase(**xargs) if type is None: data_config.type = 'proto' @@ -976,25 +983,24 @@ def ProtoData( data_config.constant_slots.extend(constant_slots) return data_config + #real data for training is actually provided by "sub_data" data providers. @config_func -def MultiData( - sub_data=[] - ): +def MultiData(sub_data=[]): data_config = DataConfig() data_config.type = 'multi' data_config.sub_data_configs.extend(sub_data) return data_config + @config_func -def Data( - type, - files=None, - feat_dim=None, - slot_dims=None, - context_len=None, - buffer_capacity=None, - **xargs): +def Data(type, + files=None, + feat_dim=None, + slot_dims=None, + context_len=None, + buffer_capacity=None, + **xargs): data_config = DataBase(**xargs) data_config.type = type @@ -1030,15 +1036,19 @@ def TestData(data_config, async_load_data=None): " Data definition") g_config.test_data_config.async_load_data = async_load_data + def parse_bilinear(bilinear, input_layer_name, bilinear_conf): - bilinear_conf.out_size_x = bilinear.out_size_x; - bilinear_conf.out_size_y = bilinear.out_size_y; - bilinear_conf.num_channels = bilinear.num_channels; + bilinear_conf.out_size_x = bilinear.out_size_x + bilinear_conf.out_size_y = bilinear.out_size_y + bilinear_conf.num_channels = bilinear.num_channels + ''' caffe_mode: compute the output size using floor instead of ceil, which is consistent of caffe and CuDNN's convention. ''' + + def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): output = (2 * padding + img_size - filter_size) / float(stride) if caffe_mode: @@ -1046,81 +1056,89 @@ def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): else: return 1 + int(math.ceil(output)) + ''' calcualte image_size based on output_size for convolution. It is the reverse function of cnn_output_size ''' + + def cnn_image_size(output_size, filter_size, padding, stride, caffe_mode): if caffe_mode: img_size = (output_size - 1) * stride + filter_size - 2 * padding else: - img_size = (output_size - 2) * stride + filter_size - 2 * padding + 1 + img_size = (output_size - 2) * stride + filter_size - 2 * padding + 1 return img_size + def parse_pool(pool, input_layer_name, pool_conf): pool_conf.pool_type = pool.pool_type - config_assert(pool.pool_type in ['max-projection', 'avg-projection', - 'cudnn-max-pool', 'cudnn-avg-pool'], - "pool-type %s is not in " + config_assert(pool.pool_type in [ + 'max-projection', 'avg-projection', 'cudnn-max-pool', 'cudnn-avg-pool' + ], "pool-type %s is not in " "['max-projection', 'avg-projection', " - "'cudnn-max-pool', 'cudnn-avg-pool']" - % pool.pool_type) + "'cudnn-max-pool', 'cudnn-avg-pool']" % pool.pool_type) pool_conf.channels = pool.channels pool_conf.size_x = pool.size_x pool_conf.stride = pool.stride pool_conf.size_y = default(pool.size_y, pool_conf.size_x) - pool_conf.stride_y = default(pool.stride_y, pool_conf.stride); + pool_conf.stride_y = default(pool.stride_y, pool_conf.stride) img_pixels = g_layer_map[input_layer_name].size / pool.channels # the img_width may be removed, # and it can be calculated automatically later. - pool_conf.img_size = default(pool.img_width, int(img_pixels ** 0.5)) + pool_conf.img_size = default(pool.img_width, int(img_pixels**0.5)) pool_conf.img_size_y = img_pixels / pool_conf.img_size config_assert(pool_conf.img_size * pool_conf.img_size_y == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (pool_conf.img_size, img_pixels)) + "Incorrect input image size %d for input image pixels %d" % + (pool_conf.img_size, img_pixels)) config_assert(not pool.start, "start is deprecated in pooling.") if pool.padding is not None: pool_conf.padding = pool.padding pool_conf.padding_y = default(pool.padding_y, pool_conf.padding) - pool_conf.output_x = cnn_output_size(pool_conf.img_size, pool_conf.size_x, - pool_conf.padding, pool_conf.stride, False) - pool_conf.output_y = cnn_output_size(pool_conf.img_size_y, pool_conf.size_y, - pool_conf.padding_y, pool_conf.stride_y, False) + pool_conf.output_x = cnn_output_size( + pool_conf.img_size, pool_conf.size_x, pool_conf.padding, + pool_conf.stride, False) + pool_conf.output_y = cnn_output_size( + pool_conf.img_size_y, pool_conf.size_y, pool_conf.padding_y, + pool_conf.stride_y, False) + def parse_spp(spp, input_layer_name, spp_conf): spp_conf.pool_type = spp.pool_type config_assert(spp.pool_type in ['max-projection', 'avg-projection'], - "pool-type %s is not in " "['max-projection', 'avg-projection']" - % spp.pool_type) + "pool-type %s is not in " + "['max-projection', 'avg-projection']" % spp.pool_type) spp_conf.pyramid_height = spp.pyramid_height spp_conf.channels = spp.channels img_pixels = g_layer_map[input_layer_name].size / spp_conf.channels - spp_conf.img_size = default(spp.img_width, int(img_pixels ** 0.5)) + spp_conf.img_size = default(spp.img_width, int(img_pixels**0.5)) spp_conf.img_size_y = img_pixels / spp_conf.img_size config_assert(spp_conf.img_size * spp_conf.img_size_y == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (spp_conf.img_size, img_pixels)) + "Incorrect input image size %d for input image pixels %d" % + (spp_conf.img_size, img_pixels)) + def parse_image(image, input_layer_name, image_conf): image_conf.channels = image.channels image_pixels = g_layer_map[input_layer_name].size / image_conf.channels - image_conf.img_size = int(image_pixels ** 0.5) - config_assert((image_conf.img_size ** 2) == image_pixels, - "Incorrect input image size %d for input image pixels %d" - % (image_conf.img_size, image_pixels)) + image_conf.img_size = int(image_pixels**0.5) + config_assert((image_conf.img_size**2) == image_pixels, + "Incorrect input image size %d for input image pixels %d" % + (image_conf.img_size, image_pixels)) + def parse_norm(norm, input_layer_name, norm_conf): norm_conf.norm_type = norm.norm_type config_assert(norm.norm_type in ['rnorm', 'cmrnorm-projection'], - "norm-type %s is not in [rnorm, 'cmrnorm-projection']" - % norm.norm_type) + "norm-type %s is not in [rnorm, 'cmrnorm-projection']" % + norm.norm_type) norm_conf.channels = norm.channels norm_conf.size = norm.size norm_conf.scale = norm.scale @@ -1128,20 +1146,23 @@ def parse_norm(norm, input_layer_name, norm_conf): norm_conf.blocked = norm.blocked img_pixels = g_layer_map[input_layer_name].size / norm.channels - norm_conf.img_size = int(img_pixels ** 0.5) - config_assert((norm_conf.img_size ** 2) == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (norm_conf.img_size, img_pixels)) + norm_conf.img_size = int(img_pixels**0.5) + config_assert((norm_conf.img_size**2) == img_pixels, + "Incorrect input image size %d for input image pixels %d" % + (norm_conf.img_size, img_pixels)) norm_conf.output_x = norm_conf.img_size if norm.norm_type in ['cmrnorm-projection']: norm_conf.scale /= norm.size else: - norm_conf.scale /= norm.size ** 2 + norm_conf.scale /= norm.size**2 + ''' caffe_mode: compute the output size using floor instead of ceil, which is consistent of caffe and CuDNN's convention. ''' + + def parse_conv(conv, input_layer_name, conv_conf, num_filters, trans=False): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y @@ -1152,36 +1173,37 @@ def parse_conv(conv, input_layer_name, conv_conf, num_filters, trans=False): conv_conf.stride_y = conv.stride_y conv_conf.groups = conv.groups conv_conf.caffe_mode = conv.caffe_mode - + if not trans: conv_conf.filter_channels = conv.channels / conv.groups img_pixels = g_layer_map[input_layer_name].size / conv.channels - print('channels=%d size=%d'%(conv.channels, - g_layer_map[input_layer_name].size)) - conv_conf.img_size = int(img_pixels ** 0.5) - config_assert((conv_conf.img_size ** 2) == img_pixels, - ("Input layer %s: Incorrect input image size %d for input " - + "image pixels %d") - % (input_layer_name, conv_conf.img_size, img_pixels)) - + print('channels=%d size=%d' % (conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.img_size = int(img_pixels**0.5) + config_assert((conv_conf.img_size**2) == img_pixels, ( + "Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") % + (input_layer_name, conv_conf.img_size, img_pixels)) + conv_conf.output_x = cnn_output_size( - conv_conf.img_size, conv_conf.filter_size, - conv_conf.padding, conv_conf.stride, conv_conf.caffe_mode) + conv_conf.img_size, conv_conf.filter_size, conv_conf.padding, + conv_conf.stride, conv_conf.caffe_mode) else: conv_conf.filter_channels = num_filters / conv.groups - + outputSize = g_layer_map[input_layer_name].size / conv.channels - print('channels=%d size=%d'%(conv.channels, - g_layer_map[input_layer_name].size)) - conv_conf.output_x = int(outputSize ** 0.5) - config_assert((conv_conf.output_x ** 2) == outputSize, - ("Input layer %s: Incorrect input image size %d for input " - + "image pixels %d") - % (input_layer_name, conv_conf.output_x, outputSize)) + print('channels=%d size=%d' % (conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.output_x = int(outputSize**0.5) + config_assert((conv_conf.output_x**2) == outputSize, ( + "Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") % + (input_layer_name, conv_conf.output_x, outputSize)) conv_conf.img_size = cnn_image_size( - conv_conf.output_x, conv_conf.filter_size, - conv_conf.padding, conv_conf.stride, conv_conf.caffe_mode) + conv_conf.output_x, conv_conf.filter_size, conv_conf.padding, + conv_conf.stride, conv_conf.caffe_mode) + def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.channels = block_expand.channels @@ -1207,27 +1229,28 @@ def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand.img_size_y, block_expand.block_y, block_expand.padding_y, block_expand.stride_y, False) + def parse_maxout(maxout, input_layer_name, maxout_conf): maxout_conf.channels = maxout.channels maxout_conf.groups = maxout.groups maxout_conf.img_size_x = maxout.img_size_x maxout_conf.img_size_y = maxout.img_size_y + # Define an evaluator @config_func def Evaluator( name, type, inputs, - chunk_scheme = None, - num_chunk_types = None, - classification_threshold = None, - positive_label = None, - dict_file = None, - result_file = None, - num_results = None, - delimited = None, - ): + chunk_scheme=None, + num_chunk_types=None, + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None, ): evaluator = g_config.model_config.evaluators.add() evaluator.type = type evaluator.name = MakeLayerNameInSubmodel(name) @@ -1256,19 +1279,20 @@ def Evaluator( if delimited is not None: evaluator.delimited = delimited + class LayerBase(object): def __init__( self, name, type, - size, # size can be 0. In this case, subclass should set it. + size, # size can be 0. In this case, subclass should set it. inputs, device=None, active_type="", drop_rate=0., coeff=None): config_assert('@' not in name, - "layer name: %s contain special character @" % name) + "layer name: %s contain special character @" % name) global g_current_submodel name = MakeLayerNameInSubmodel(name) @@ -1307,8 +1331,8 @@ class LayerBase(object): if type_of(input) == str: input_layer_name = input input_config = Input( - input_layer_name = input, - parameter_name = gen_parameter_name(name, input_index)) + input_layer_name=input, + parameter_name=gen_parameter_name(name, input_index)) input_layer_name = input_config.input_layer_name elif isinstance(input, Input): input_layer_name = input.input_layer_name @@ -1317,16 +1341,15 @@ class LayerBase(object): input_config.parameter_name = \ gen_parameter_name(name, input_index) elif isinstance(input, Operator): - self.operators.append(input); + self.operators.append(input) input.operator_conf.input_indices.append(input_index) input_config = Input(input.input_layer_names[0]) input_layer_name = input_config.input_layer_name else: - raise ValueError( - 'Wrong type for inputs: %s' % type_of(input)) + raise ValueError('Wrong type for inputs: %s' % type_of(input)) config_assert(input_layer_name in g_layer_map, - "Unknown input layer '%s' for layer %s" - % (input_layer_name, name)) + "Unknown input layer '%s' for layer %s" % + (input_layer_name, name)) self.inputs[input_index] = input_config layer_input = self.config.inputs.add() layer_input.input_layer_name = input_config.input_layer_name @@ -1338,26 +1361,26 @@ class LayerBase(object): g_current_submodel.layer_names.append(self.config.name) - def get_input_layer(self, input_index): return g_layer_map[self.config.inputs[input_index].input_layer_name] # will return the bias created if not *for_self* def create_bias_parameter( self, - bias, # True/False or BiasCfg + bias, # True/False or BiasCfg size, - dims = None, - for_self = True, # whether create bias for layer self - ): + dims=None, + for_self=True, # whether create bias for layer self + ): if size == 0: return if dims is None: dims = [1, size] - config_assert(type_of(bias) == bool or type_of(bias) == Bias, - 'Incorrect type for bias: %s' % type_of(bias)) + config_assert( + type_of(bias) == bool or type_of(bias) == Bias, + 'Incorrect type for bias: %s' % type_of(bias)) if type_of(bias) == bool: if bias: @@ -1372,7 +1395,8 @@ class LayerBase(object): Parameter( bias.parameter_name, size, - self.config.device if self.config.HasField('device') else None, + self.config.device + if self.config.HasField('device') else None, dims, bias.learning_rate, bias.momentum, @@ -1384,22 +1408,21 @@ class LayerBase(object): initial_smart=bias.initial_smart, num_batches_regularization=bias.num_batches_regularization, sparse_remote_update=bias.sparse_remote_update, - gradient_clipping_threshold=bias.gradient_clipping_threshold, + gradient_clipping_threshold=bias. + gradient_clipping_threshold, is_static=bias.is_static, - is_shared=bias.is_shared, - ) + is_shared=bias.is_shared, ) if for_self: self.config.bias_parameter_name = bias.parameter_name else: return bias.parameter_name - def create_input_parameter( - self, - input_index, - size, - dims=None, - sparse = None, - format = None): + def create_input_parameter(self, + input_index, + size, + dims=None, + sparse=None, + format=None): if dims is None: # TODO(yuyang18): print warning and callstack here! dims = list() @@ -1414,12 +1437,12 @@ class LayerBase(object): if input_config.parameter_name in g_parameter_map: para = g_parameter_map[input_config.parameter_name] - config_assert(size == para.size, ('Shared parameter "%s" does not ' - + 'have same size: %s vs. %s') + config_assert(size == para.size, ( + 'Shared parameter "%s" does not ' + 'have same size: %s vs. %s') % (input_config.parameter_name, para.size, size)) - config_assert(dims == para.dims, ('Shared parameter "%s" does not ' - + 'have same dims: %s vs. %s') + config_assert(dims == para.dims, ( + 'Shared parameter "%s" does not ' + 'have same dims: %s vs. %s') % (input_config.parameter_name, para.dims, dims)) return @@ -1439,13 +1462,13 @@ class LayerBase(object): num_batches_regularization=input_config.num_batches_regularization, sparse_remote_update=input_config.sparse_remote_update, sparse_update=input_config.sparse_update, - gradient_clipping_threshold=input_config.gradient_clipping_threshold, + gradient_clipping_threshold=input_config. + gradient_clipping_threshold, sparse=sparse, format=format, is_static=input_config.is_static, is_shared=input_config.is_shared, - update_hooks=input_config.update_hooks - ) + update_hooks=input_config.update_hooks) def set_layer_size(self, size): if self.config.size == 0: @@ -1455,27 +1478,18 @@ class LayerBase(object): 'Different inputs result in' + 'different layer size at layer %s' % self.config.name) + @config_layer('multi_class_cross_entropy_with_selfnorm') class MultiClassCrossEntropySelfNormCostLayer(LayerBase): - def __init__( - self, - name, - inputs, - softmax_selfnorm_alpha=0.1, - **xargs): - super(MultiClassCrossEntropySelfNormCostLayer, self).__init__(name, - 'multi_class_cross_entropy_with_selfnorm', 0, inputs, **xargs) + def __init__(self, name, inputs, softmax_selfnorm_alpha=0.1, **xargs): + super(MultiClassCrossEntropySelfNormCostLayer, self).__init__( + name, 'multi_class_cross_entropy_with_selfnorm', 0, inputs, **xargs) self.config.softmax_selfnorm_alpha = softmax_selfnorm_alpha + @config_layer('fc') class FCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - bias=True, - **xargs): + def __init__(self, name, size, inputs, bias=True, **xargs): super(FCLayer, self).__init__(name, 'fc', size, inputs=inputs, **xargs) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) @@ -1489,22 +1503,23 @@ class FCLayer(LayerBase): else: sparse = None - self.create_input_parameter(input_index, psize, dims, sparse, format) + self.create_input_parameter(input_index, psize, dims, sparse, + format) self.create_bias_parameter(bias, self.config.size) + @config_layer('selective_fc') class SelectiveFCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - bias=True, - selective_fc_pass_generation=False, - has_selected_colums=True, - selective_fc_full_mul_ratio=0.02, - selective_fc_parallel_plain_mul_thread_num=None, - **xargs): + def __init__(self, + name, + size, + inputs, + bias=True, + selective_fc_pass_generation=False, + has_selected_colums=True, + selective_fc_full_mul_ratio=0.02, + selective_fc_parallel_plain_mul_thread_num=None, + **xargs): super(SelectiveFCLayer, self).__init__( name, 'selective_fc', size, inputs=inputs, **xargs) # user MUST know if selctive fc is used in training, @@ -1525,8 +1540,8 @@ class SelectiveFCLayer(LayerBase): input_num = len(self.inputs) if has_selected_colums: config_assert(input_num >= 2, - ("if indices of selected columns are not specified, " - "selective_fc Layer has at least two inputs")) + ("if indices of selected columns are not specified, " + "selective_fc Layer has at least two inputs")) input_num -= 1 for input_index in xrange(input_num): @@ -1539,26 +1554,23 @@ class SelectiveFCLayer(LayerBase): if sparse: psize = self.inputs[input_index].nnz - self.create_input_parameter( - input_index, psize, dims, sparse, format) + self.create_input_parameter(input_index, psize, dims, sparse, + format) self.create_bias_parameter(bias, self.config.size) + @config_layer('print') class PrintLayer(LayerBase): - def __init__( - self, - name, - inputs): + def __init__(self, name, inputs): super(PrintLayer, self).__init__(name, 'print', 0, inputs) + @config_layer('data') class DataLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): - super(DataLayer, self).__init__(name, 'data' , size, inputs=[], device=device) + def __init__(self, name, size, device=None): + super(DataLayer, self).__init__( + name, 'data', size, inputs=[], device=device) + ''' DataNormLayer: A layer for data normalization @@ -1586,14 +1598,11 @@ Note: min-max: y = (x-min)/(max-min) decimal-scaling: y = x/10^j, where j is the smallest integer such that max(|y|)<1 ''' + + @config_layer('data_norm') class DataNormLayer(LayerBase): - def __init__( - self, - name, - inputs, - data_norm_strategy="z-score", - device=None): + def __init__(self, name, inputs, data_norm_strategy="z-score", device=None): super(DataNormLayer, self).__init__( name, 'data_norm', 0, inputs=inputs, device=device) self.config.data_norm_strategy = data_norm_strategy @@ -1605,15 +1614,12 @@ class DataNormLayer(LayerBase): self.inputs[0].is_static = True self.create_input_parameter(0, para_size, para_dims) + @config_layer('prelu') class ParameterReluLayer(LayerBase): layer_type = 'prelu' - def __init__( - self, - name, - inputs, - partial_sum = 1, - **args): + + def __init__(self, name, inputs, partial_sum=1, **args): super(ParameterReluLayer, self).__init__( name, self.layer_type, 0, inputs=inputs, **args) config_assert(len(self.inputs) == 1) @@ -1622,17 +1628,18 @@ class ParameterReluLayer(LayerBase): self.set_layer_size(input_layer.size) self.create_input_parameter(0, input_layer.size / partial_sum) + @config_layer('conv') class ConvLayerBase(LayerBase): layer_type = 'conv' - def __init__( - self, - name, - inputs=[], - bias=True, - num_filters=None, - shared_biases=False, - **xargs): + + def __init__(self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): super(ConvLayerBase, self).__init__( name, self.layer_type, 0, inputs=inputs, **xargs) @@ -1649,7 +1656,7 @@ class ConvLayerBase(LayerBase): config_assert(use_gpu, "cudnn_conv only support GPU") if (use_gpu == 1 and self.layer_type != "exconv" and - (parallel_nn == 0 or self.config.device > -1)): + (parallel_nn == 0 or self.config.device > -1)): self.layer_type = "cudnn_conv" else: self.layer_type = "exconv" @@ -1661,17 +1668,14 @@ class ConvLayerBase(LayerBase): for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_conv( - self.inputs[input_index].conv, - input_layer.name, - self.config.inputs[input_index].conv_conf, - num_filters) + parse_conv(self.inputs[input_index].conv, input_layer.name, + self.config.inputs[input_index].conv_conf, num_filters) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) print("output size for %s is %d " % (name, conv_conf.output_x)) self.create_input_parameter(input_index, psize) self.set_layer_size( - (conv_conf.output_x ** 2) * self.config.num_filters) + (conv_conf.output_x**2) * self.config.num_filters) psize = self.config.size if shared_biases: @@ -1682,10 +1686,12 @@ class ConvLayerBase(LayerBase): return self.config.num_filters * conv_conf.filter_channels \ * (conv_conf.filter_size * conv_conf.filter_size_y) + @config_layer('exconv') class ConvLayer(ConvLayerBase): layer_type = 'exconv' + @config_layer('cudnn_conv') class ConvLayer(ConvLayerBase): layer_type = 'cudnn_conv' @@ -1694,14 +1700,14 @@ class ConvLayer(ConvLayerBase): @config_layer('convt') class ConvTransLayerBase(LayerBase): layer_type = 'convt' - def __init__( - self, - name, - inputs=[], - bias=True, - num_filters=None, - shared_biases=False, - **xargs): + + def __init__(self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): super(ConvTransLayerBase, self).__init__( name, self.layer_type, 0, inputs=inputs, **xargs) @@ -1732,7 +1738,7 @@ class ConvTransLayerBase(LayerBase): print("output size for %s is %d " % (name, conv_conf.output_x)) self.create_input_parameter(input_index, psize) self.set_layer_size( - (conv_conf.img_size ** 2) * self.config.num_filters) + (conv_conf.img_size**2) * self.config.num_filters) psize = self.config.size if shared_biases: @@ -1743,85 +1749,76 @@ class ConvTransLayerBase(LayerBase): return conv_conf.channels * conv_conf.filter_channels \ * (conv_conf.filter_size * conv_conf.filter_size_y) + @config_layer('exconvt') class ConvTransLayer(ConvTransLayerBase): layer_type = 'exconvt' + @config_layer('norm') class NormLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(NormLayer, self).__init__(name, 'norm', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(NormLayer, self).__init__( + name, 'norm', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_norm( - self.inputs[input_index].norm, - input_layer.name, - self.config.inputs[input_index].norm_conf) + parse_norm(self.inputs[input_index].norm, input_layer.name, + self.config.inputs[input_index].norm_conf) norm_conf = self.config.inputs[input_index].norm_conf - self.set_layer_size((norm_conf.output_x ** 2) * norm_conf.channels) + self.set_layer_size((norm_conf.output_x**2) * norm_conf.channels) + @config_layer('pool') class PoolLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(PoolLayer, self).__init__(name, 'pool', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(PoolLayer, self).__init__( + name, 'pool', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_pool( - self.inputs[input_index].pool, - input_layer.name, - self.config.inputs[input_index].pool_conf) + parse_pool(self.inputs[input_index].pool, input_layer.name, + self.config.inputs[input_index].pool_conf) pool_conf = self.config.inputs[input_index].pool_conf - print("output size for %s is %d*%d " % ( - name, pool_conf.output_y, pool_conf.output_x)) - self.set_layer_size((pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) + print("output size for %s is %d*%d " % (name, pool_conf.output_y, + pool_conf.output_x)) + self.set_layer_size( + (pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) + @config_layer('spp') class SpatialPyramidPoolLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(SpatialPyramidPoolLayer, self).__init__(name, 'spp', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(SpatialPyramidPoolLayer, self).__init__( + name, 'spp', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_spp( - self.inputs[input_index].spp, - input_layer.name, - self.config.inputs[input_index].spp_conf) + parse_spp(self.inputs[input_index].spp, input_layer.name, + self.config.inputs[input_index].spp_conf) spp_conf = self.config.inputs[input_index].spp_conf output_size = (pow(4, spp_conf.pyramid_height) - 1) / (4 - 1) print("output size for %s is %d " % (name, output_size)) self.set_layer_size(output_size * spp_conf.channels) + @config_layer('batch_norm') class BatchNormLayer(LayerBase): layer_type = 'batch_norm' - def __init__( - self, - name, - inputs, - active_type="linear", - bias=True, - device=None, - use_global_stats=True, - moving_average_fraction=0.9, - batch_norm_type=None, - **xargs): + + def __init__(self, + name, + inputs, + active_type="linear", + bias=True, + device=None, + use_global_stats=True, + moving_average_fraction=0.9, + batch_norm_type=None, + **xargs): if inputs is None: inputs = [] elif not isinstance(inputs, list): inputs = [inputs] - config_assert(len(inputs) == 1, - "BatchNormLayer must have one and only one input") + config_assert( + len(inputs) == 1, "BatchNormLayer must have one and only one input") # Create Input for moving mean and std, # in batch normalization layer. # These paras no need to update, so set is_static is true. @@ -1830,12 +1827,13 @@ class BatchNormLayer(LayerBase): use_gpu = bool(int(g_command_config_args.get("use_gpu", 0))) is_shared = True if not use_gpu else False for i in xrange(2): - inputs.append(Input(inputs[0].input_layer_name, - initial_std=0.0, - initial_mean=0.0, - is_static=True, - is_shared=is_shared, - )) + inputs.append( + Input( + inputs[0].input_layer_name, + initial_std=0.0, + initial_mean=0.0, + is_static=True, + is_shared=is_shared, )) parallel_nn = bool(int(g_command_config_args.get("parallel_nn", 0))) cudnn_version = int(g_command_config_args.get("cudnn_version", 0)) @@ -1845,21 +1843,25 @@ class BatchNormLayer(LayerBase): ((not parallel_nn) or self.config.device > -1) and \ cudnn_version >= 4007 self.layer_type = "cudnn_batch_norm" if use_cudnn else "batch_norm" - super(BatchNormLayer, self).__init__(name, self.layer_type, 0, - active_type=active_type, - inputs=inputs, device=device, **xargs) + super(BatchNormLayer, self).__init__( + name, + self.layer_type, + 0, + active_type=active_type, + inputs=inputs, + device=device, + **xargs) if use_global_stats is not None: self.config.use_global_stats = use_global_stats if moving_average_fraction is not None: self.config.moving_average_fraction = moving_average_fraction - input_layer= self.get_input_layer(0) - parse_image(self.inputs[0].image, - input_layer.name, + input_layer = self.get_input_layer(0) + parse_image(self.inputs[0].image, input_layer.name, self.config.inputs[0].image_conf) image_conf = self.config.inputs[0].image_conf - self.set_layer_size((image_conf.img_size ** 2) * image_conf.channels) + self.set_layer_size((image_conf.img_size**2) * image_conf.channels) psize = self.calc_parameter_size(image_conf) dims = [1, psize] @@ -1872,75 +1874,74 @@ class BatchNormLayer(LayerBase): def calc_parameter_size(self, image_conf): return image_conf.channels + @config_layer('trans') class TransLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(TransLayer, self).__init__(name, 'trans', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'TransLayer must have one and only one input') + def __init__(self, name, inputs, device=None): + super(TransLayer, self).__init__( + name, 'trans', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, + 'TransLayer must have one and only one input') self.set_layer_size(self.get_input_layer(0).size) + @config_layer('resize') class ResizeLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): - super(ResizeLayer, self).__init__(name, 'resize', size=size, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'ResizeLayer must have one and only one input') + def __init__(self, name, size, inputs, device=None): + super(ResizeLayer, self).__init__( + name, 'resize', size=size, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, + 'ResizeLayer must have one and only one input') + @config_layer('blockexpand') class BlockExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(BlockExpandLayer, self).__init__(name, 'blockexpand', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(BlockExpandLayer, self).__init__( + name, 'blockexpand', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_block_expand(self.inputs[input_index].block_expand, - input_layer.name, + parse_block_expand( + self.inputs[input_index].block_expand, input_layer.name, self.config.inputs[input_index].block_expand_conf) - block_expand_conf = self.config.inputs[input_index].block_expand_conf - self.set_layer_size(block_expand_conf.block_x * block_expand_conf.block_y - * block_expand_conf.channels) + block_expand_conf = self.config.inputs[ + input_index].block_expand_conf + self.set_layer_size(block_expand_conf.block_x * + block_expand_conf.block_y * + block_expand_conf.channels) + @config_layer('maxout') class MaxOutLayer(LayerBase): - def __init__( - self, - name, - inputs, - **xargs): - super(MaxOutLayer, self).__init__(name, 'maxout', 0, inputs=inputs, **xargs) + def __init__(self, name, inputs, **xargs): + super(MaxOutLayer, self).__init__( + name, 'maxout', 0, inputs=inputs, **xargs) input_layer = self.get_input_layer(0) - parse_maxout(self.inputs[0].maxout, - input_layer.name, + parse_maxout(self.inputs[0].maxout, input_layer.name, self.config.inputs[0].maxout_conf) maxout_conf = self.config.inputs[0].maxout_conf - self.set_layer_size(g_layer_map[input_layer.name].size / maxout_conf.groups) + self.set_layer_size(g_layer_map[input_layer.name].size / + maxout_conf.groups) + # key: cost type # value: cost class g_cost_map = {} + # define a cost layer without any parameters def define_cost(class_name, cost_type): def init(cls, name, inputs, device=None, coeff=1.): - super(type(cls), cls).__init__(name, cost_type, 1, inputs, device=device, coeff=coeff) + super(type(cls), cls).__init__( + name, cost_type, 1, inputs, device=device, coeff=coeff) - cls = type(class_name, (LayerBase,), dict(__init__=init)) + cls = type(class_name, (LayerBase, ), dict(__init__=init)) global g_cost_map g_cost_map[cost_type] = cls + define_cost('MultiClassCrossEntropy', 'multi-class-cross-entropy') define_cost('RankingCost', 'rank-cost') define_cost('AucValidation', 'auc-validation') @@ -1951,19 +1952,15 @@ define_cost('SoftBinaryClassCrossEntropy', 'soft_binary_class_cross_entropy') define_cost('HuberTwoClass', 'huber') define_cost('SumCost', 'sum_cost') + @config_layer('hsigmoid') class HierarchicalSigmoidLayer(LayerBase): - def __init__( - self, - name, - num_classes, - inputs, - device=None, - bias=True): + def __init__(self, name, num_classes, inputs, device=None, bias=True): super(HierarchicalSigmoidLayer, self).__init__( name, 'hsigmoid', 1, inputs=inputs, device=device) - config_assert(len(self.inputs) >= 2, - 'HierarchicalSigmoidLayer must have at least 2 inputs') + config_assert( + len(self.inputs) >= 2, + 'HierarchicalSigmoidLayer must have at least 2 inputs') self.config.num_classes = num_classes for input_index in xrange(len(self.inputs) - 1): input_layer = self.get_input_layer(input_index) @@ -1972,6 +1969,7 @@ class HierarchicalSigmoidLayer(LayerBase): self.create_input_parameter(input_index, psize, dims) self.create_bias_parameter(bias, num_classes - 1) + ''' lambdaCost for lambdaRank LTR approach @@ -1996,59 +1994,57 @@ Usage: max_sort_size can be greater than the size of a list, in which case the algorithm will sort the entire list to get gradient. ''' + + @config_layer('lambda_cost') class LambdaCost(LayerBase): - def __init__( - self, - name, - inputs, - NDCG_num = 5, - max_sort_size = -1, - device=None): + def __init__(self, name, inputs, NDCG_num=5, max_sort_size=-1, device=None): super(LambdaCost, self).__init__( name, 'lambda_cost', 1, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'lambdaCost must have 2 inputs') + config_assert(len(self.inputs) == 2, 'lambdaCost must have 2 inputs') self.config.NDCG_num = NDCG_num if max_sort_size != -1: - config_assert(NDCG_num <= max_sort_size, - 'NDCG_num must be less than or equal to max_sort_size') + config_assert( + NDCG_num <= max_sort_size, + 'NDCG_num must be less than or equal to max_sort_size') self.config.max_sort_size = max_sort_size + @config_layer('nce') class NCELayer(LayerBase): - def __init__( - self, - name, - num_classes, - inputs, - num_neg_samples=10, - neg_sampling_dist=None, - bias=True, - **xargs): + def __init__(self, + name, + num_classes, + inputs, + num_neg_samples=10, + neg_sampling_dist=None, + bias=True, + **xargs): super(NCELayer, self).__init__(name, 'nce', 1, inputs=inputs, **xargs) - config_assert(len(self.inputs) >= 2, - 'NCELayer must have at least 2 inputs') + config_assert( + len(self.inputs) >= 2, 'NCELayer must have at least 2 inputs') self.config.num_classes = num_classes if neg_sampling_dist is not None: - config_assert(len(neg_sampling_dist) == num_classes, - 'len(neg_sampling_dist)(%s) is not same as num_classes (%s)' - % (len(neg_sampling_dist), num_classes)) + config_assert( + len(neg_sampling_dist) == num_classes, + 'len(neg_sampling_dist)(%s) is not same as num_classes (%s)' % + (len(neg_sampling_dist), num_classes)) s = sum(neg_sampling_dist) - config_assert(abs(s - 1) < 1e-5, - 'The sum of neg_sampling_dist (%s) is not 1' % s) + config_assert( + abs(s - 1) < 1e-5, + 'The sum of neg_sampling_dist (%s) is not 1' % s) self.config.neg_sampling_dist.extend(neg_sampling_dist) self.config.num_neg_samples = num_neg_samples num_real_inputs = len(self.inputs) - 1 - input_layer = self.get_input_layer(num_real_inputs) + input_layer = self.get_input_layer(num_real_inputs) config_assert(input_layer.type == 'data', 'Expecting the last input layer of an nce layer to be ' 'a data layer') - if (num_real_inputs > 1 and input_layer.size == 1 - and self.get_input_layer(num_real_inputs - 1).type == 'data'): + if (num_real_inputs > 1 and input_layer.size == 1 and + self.get_input_layer(num_real_inputs - 1).type == 'data'): # This input layer is assumed to be a sample weight layer num_real_inputs -= 1 @@ -2062,105 +2058,82 @@ class NCELayer(LayerBase): @config_layer('addto') class AddToLayer(LayerBase): - def __init__( - self, - name, - inputs, - bias=True, - **xargs): + def __init__(self, name, inputs, bias=True, **xargs): super(AddToLayer, self).__init__( name, 'addto', 0, inputs=inputs, **xargs) - config_assert(len(inputs) > 0, - 'inputs cannot be empty for AddToLayer') + config_assert(len(inputs) > 0, 'inputs cannot be empty for AddToLayer') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('agent') class AgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): - super(AgentLayer, self).__init__(name, 'agent', size, inputs=[], device=device) + def __init__(self, name, size, device=None): + super(AgentLayer, self).__init__( + name, 'agent', size, inputs=[], device=device) + @config_layer('sequence_agent') class SequenceAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceAgentLayer, self).__init__( name, 'sequence_agent', size, inputs=[], device=device) + @config_layer('gather_agent') class GatherAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(GatherAgentLayer, self).__init__( name, 'gather_agent', size, inputs=[], device=device) + @config_layer('scatter_agent') class ScatterAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(ScatterAgentLayer, self).__init__( name, 'scatter_agent', size, inputs=[], device=device) + @config_layer('sequence_gather_agent') class SequenceGatherAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceGatherAgentLayer, self).__init__( - name, 'sequence_gather_agent', size, inputs=[], device=device) + name, 'sequence_gather_agent', size, inputs=[], device=device) + @config_layer('sequence_scatter_agent') class SequenceScatterAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceScatterAgentLayer, self).__init__( - name, 'sequence_scatter_agent', size, inputs=[], device=device) + name, 'sequence_scatter_agent', size, inputs=[], device=device) + @config_layer('multiplex') class MultiplexLayer(LayerBase): - def __init__( - self, - name, - inputs, - size, - device=None): - super(MultiplexLayer, self).__init__(name, 'multiplex', size, inputs=inputs, device=device) - config_assert(len(inputs) > 2, - 'MultiplexLayer should have more than 2 inputs.') + def __init__(self, name, inputs, size, device=None): + super(MultiplexLayer, self).__init__( + name, 'multiplex', size, inputs=inputs, device=device) + config_assert( + len(inputs) > 2, 'MultiplexLayer should have more than 2 inputs.') for i in range(1, len(inputs)): - config_assert(self.get_input_layer(i).size == size, - "All the input layers except the first one should" - "have the same size as the MultiplexLayer.") + config_assert( + self.get_input_layer(i).size == size, + "All the input layers except the first one should" + "have the same size as the MultiplexLayer.") + @config_func -def Link(name, - has_subseq=False, - ): +def Link( + name, + has_subseq=False, ): link_config = LinkConfig() link_config.link_name = name link_config.has_subseq = has_subseq return link_config + # memory for recurrent layer group. # *name* and *size* are actual layer's name and size. # will return name of the memory, @@ -2175,43 +2148,46 @@ def Link(name, # can only be initailized by a *boot_layer* which is a sequence. # @config_func -def Memory(name, - size, - is_sequence=False, - boot_layer=None, - boot_bias=False, - boot_bias_active_type="", - boot_with_const_id=None, - ): +def Memory( + name, + size, + is_sequence=False, + boot_layer=None, + boot_bias=False, + boot_bias_active_type="", + boot_with_const_id=None, ): agent_name = name + "+delay1" if is_sequence: agent_layer = SequenceAgentLayer(agent_name, size) else: agent_layer = AgentLayer(agent_name, size) config_assert(g_current_submodel.is_recurrent_layer_group, - 'Memory should be used in recurrent layer group only') + 'Memory should be used in recurrent layer group only') memory = g_current_submodel.memories.add() memory.layer_name = MakeLayerNameInSubmodel(name) memory.link_name = MakeLayerNameInSubmodel(agent_name) memory.is_sequence = is_sequence - options = sum((boot_layer is not None, - bool(boot_bias), + options = sum((boot_layer is not None, bool(boot_bias), boot_with_const_id is not None)) - config_assert(options <= 1, - 'take one option at most from boot_layer, boot_bias, or boot_with_const_id') + config_assert( + options <= 1, + 'take one option at most from boot_layer, boot_bias, or boot_with_const_id' + ) if boot_layer is not None: boot_layer = MakeLayerNameInParentSubmodel(boot_layer) config_assert(boot_layer in g_layer_map, - 'boot_layer "%s" does not correspond to a layer name' % boot_layer) + 'boot_layer "%s" does not correspond to a layer name' % + boot_layer) memory.boot_layer_name = boot_layer elif boot_bias: memory.boot_bias_parameter_name = agent_layer.create_bias_parameter( - boot_bias, size, for_self = False) + boot_bias, size, for_self=False) memory.boot_bias_active_type = boot_bias_active_type elif boot_with_const_id is not None: memory.boot_with_const_id = boot_with_const_id return agent_name + # Generator for recurrent layer group, to use it: # 1. define a id layer as output of layer group # 2. define a memory of this id layer, and assign a boot id(begin of sequence) @@ -2223,11 +2199,10 @@ def Memory(name, @config_func def Generator( max_num_frames, - eos_layer_name = "eos_check", - num_results_per_sample = 1, - beam_size = 1, - log_prob = None, - ): + eos_layer_name="eos_check", + num_results_per_sample=1, + beam_size=1, + log_prob=None, ): generator_config = GeneratorConfig() generator_config.max_num_frames = max_num_frames generator_config.eos_layer_name = eos_layer_name @@ -2237,60 +2212,55 @@ def Generator( generator_config.log_prob = log_prob return generator_config + @config_layer('expand') class ExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - trans_type='non-seq', - device=None, - bias=False): - super(ExpandLayer, self).__init__( - name, 'expand', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'ExpandLayer takes 2 and only 2 inputs') - self.config.trans_type = trans_type - for input_index in xrange(len(self.inputs)): - input_layer = self.get_input_layer(input_index) - self.set_layer_size(self.get_input_layer(0).size) - self.create_bias_parameter(bias, self.config.size) + def __init__(self, + name, + inputs, + trans_type='non-seq', + device=None, + bias=False): + super(ExpandLayer, self).__init__( + name, 'expand', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 2, 'ExpandLayer takes 2 and only 2 inputs') + self.config.trans_type = trans_type + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + self.set_layer_size(self.get_input_layer(0).size) + self.create_bias_parameter(bias, self.config.size) + @config_layer('featmap_expand') class FeatMapExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None, - num_filters=None, - bias=False): - super(FeatMapExpandLayer, self).__init__( - name, 'featmap_expand', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'ExpandLayer takes 1 and only 1 inputs') - if num_filters is not None: + def __init__(self, name, inputs, device=None, num_filters=None, bias=False): + super(FeatMapExpandLayer, self).__init__( + name, 'featmap_expand', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, 'ExpandLayer takes 1 and only 1 inputs') + if num_filters is not None: self.config.num_filters = num_filters - else: + else: logger.fatal("FeatMapExpandLayer must specify num_filters.") - self.set_layer_size(self.get_input_layer(0).size * num_filters) + self.set_layer_size(self.get_input_layer(0).size * num_filters) @config_layer('max') class MaxLayer(LayerBase): - def __init__( - self, - name, - inputs, - trans_type='non-seq', - active_type='linear', - device=None, - bias=False, - output_max_index=None): - super(MaxLayer, self).__init__(name, 'max', 0, inputs=inputs, device=device) + def __init__(self, + name, + inputs, + trans_type='non-seq', + active_type='linear', + device=None, + bias=False, + output_max_index=None): + super(MaxLayer, self).__init__( + name, 'max', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') - self.config.trans_type = trans_type - self.config.active_type = active_type + self.config.trans_type = trans_type + self.config.active_type = active_type for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2301,12 +2271,7 @@ class MaxLayer(LayerBase): @config_layer('maxid') class MaxIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - beam_size=None, - device=None): + def __init__(self, name, inputs, beam_size=None, device=None): super(MaxIdLayer, self).__init__( name, 'maxid', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxIdLayer must have 1 input') @@ -2324,37 +2289,39 @@ class MaxIdLayer(LayerBase): @config_layer('eos_id') class EosIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - eos_id, - device=None): + def __init__(self, name, inputs, eos_id, device=None): super(EosIdLayer, self).__init__( name, 'eos_id', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'EosIdLayer must have 1 input') - self.set_layer_size(2) # boolean output + self.set_layer_size(2) # boolean output self.config.eos_id = eos_id + @config_layer('seqlastins') class SequenceLastInstanceLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - trans_type='non-seq', - device=None, - bias=False): - super(SequenceLastInstanceLayer, self).__init__(name, 'seqlastins', - 0, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') - self.config.trans_type = trans_type + def __init__(self, + name, + inputs, + active_type='linear', + trans_type='non-seq', + device=None, + bias=False): + super(SequenceLastInstanceLayer, self).__init__( + name, + 'seqlastins', + 0, + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') + self.config.trans_type = trans_type for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('seqfirstins') class SequenceFirstInstanceLayer(SequenceLastInstanceLayer): def __init__( @@ -2364,167 +2331,163 @@ class SequenceFirstInstanceLayer(SequenceLastInstanceLayer): active_type='linear', trans_type='non-seq', device=None, - bias=False, - ): - super(SequenceFirstInstanceLayer, self).__init__(name, - inputs=inputs, active_type=active_type, device=device, bias=bias) - self.config.trans_type = trans_type + bias=False, ): + super(SequenceFirstInstanceLayer, self).__init__( + name, + inputs=inputs, + active_type=active_type, + device=device, + bias=bias) + self.config.trans_type = trans_type self.config.select_first = True + @config_layer('seqconcat') class SequenceConcatLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - device=None, - bias=False): - super(SequenceConcatLayer, self).__init__(name, 'seqconcat', - 0, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') + def __init__(self, + name, + inputs, + active_type='linear', + device=None, + bias=False): + super(SequenceConcatLayer, self).__init__( + name, + 'seqconcat', + 0, + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('seqreshape') class SequenceReshapeLayer(LayerBase): - def __init__( - self, - name, + def __init__(self, + name, + size, + inputs, + active_type='linear', + device=None, + bias=False): + super(SequenceReshapeLayer, self).__init__( + name, + 'seqreshape', size, - inputs, - active_type='linear', - device=None, - bias=False): - super(SequenceReshapeLayer, self).__init__(name, 'seqreshape', - size, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') self.set_layer_size(size) self.create_bias_parameter(bias, size) + @config_layer('subseq') class SubSequenceLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - device=None, - bias=False): - super(SubSequenceLayer, self).__init__(name, 'subseq', - 0, inputs=inputs, device=device, active_type=active_type) + def __init__(self, + name, + inputs, + active_type='linear', + device=None, + bias=False): + super(SubSequenceLayer, self).__init__( + name, + 'subseq', + 0, + inputs=inputs, + device=device, + active_type=active_type) config_assert(len(inputs) == 3, 'SubSequenceLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) size = input_layer0.size self.set_layer_size(size) self.create_bias_parameter(bias, size) + @config_layer('out_prod') class OuterProdLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(OuterProdLayer, self).__init__(name, 'out_prod', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(OuterProdLayer, self).__init__( + name, 'out_prod', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'OuterProdLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer0.size * input_layer1.size) + @config_layer('power') class PowerLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(PowerLayer, self).__init__(name, 'power', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(PowerLayer, self).__init__( + name, 'power', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'PowerLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) input_layer0 = self.get_input_layer(0) - config_assert(1==input_layer0.size, - 'The left input is the exponent and should be of size 1') + config_assert(1 == input_layer0.size, + 'The left input is the exponent and should be of size 1') + @config_layer('slope_intercept') class SlopeInterceptLayer(LayerBase): - def __init__( - self, - name, - inputs, - slope=1.0, - intercept=0.0, - device=None): - super(SlopeInterceptLayer, self).__init__(name, 'slope_intercept', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, slope=1.0, intercept=0.0, device=None): + super(SlopeInterceptLayer, self).__init__( + name, 'slope_intercept', 0, inputs=inputs, device=device) self.config.slope = slope self.config.intercept = intercept config_assert(len(inputs) == 1, 'SlopeInterceptLayer must have 1 input') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('scaling') class ScalingLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(ScalingLayer, self).__init__(name, 'scaling', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(ScalingLayer, self).__init__( + name, 'scaling', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ScalingLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) input_layer0 = self.get_input_layer(0) - config_assert(1==input_layer0.size, - 'The left input should be of size 1') + config_assert(1 == input_layer0.size, + 'The left input should be of size 1') + @config_layer('conv_shift') class ConvShiftLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(ConvShiftLayer, self).__init__(name, 'conv_shift', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(ConvShiftLayer, self).__init__( + name, 'conv_shift', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ConvShiftLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('convex_comb') class ConvexCombinationLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): + def __init__(self, name, size, inputs, device=None): super(ConvexCombinationLayer, self).__init__( - name, 'convex_comb', size, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'ConvexCombinationLayer must have 2 inputs') + name, 'convex_comb', size, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 2, 'ConvexCombinationLayer must have 2 inputs') config_assert( size * self.get_input_layer(0).size == self.get_input_layer(1).size, 'Wrong input size for ConvexCombinationLayer') self.set_layer_size(size) + @config_layer('interpolation') class InterpolationLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(InterpolationLayer, self).__init__( name, 'interpolation', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 3, - 'InterpolationLayer must have 3 inputs') + config_assert( + len(self.inputs) == 3, 'InterpolationLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) input_layer2 = self.get_input_layer(2) @@ -2533,64 +2496,51 @@ class InterpolationLayer(LayerBase): config_assert(input_layer1.size == input_layer2.size, 'the two vector inputs should be of the same size') + @config_layer('bilinear_interp') class BilinearInterpLayer(LayerBase): - def __init__( - self, - name, - inputs, - **xargs): + def __init__(self, name, inputs, **xargs): super(BilinearInterpLayer, self).__init__( name, 'bilinear_interp', 0, inputs=inputs, **xargs) input_layer = self.get_input_layer(0) - parse_bilinear(self.inputs[0].bilinear_interp, - input_layer.name, - self.config.inputs[0].bilinear_interp_conf); + parse_bilinear(self.inputs[0].bilinear_interp, input_layer.name, + self.config.inputs[0].bilinear_interp_conf) conf = self.inputs[0].bilinear_interp - self.set_layer_size(conf.out_size_x * conf.out_size_y * conf.num_channels) + self.set_layer_size(conf.out_size_x * conf.out_size_y * + conf.num_channels) + @config_layer('sum_to_one_norm') class SumToOneNormLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(SumToOneNormLayer, self).__init__( - name, 'sum_to_one_norm', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'SumToOneNormLayer must have 1 input') + name, 'sum_to_one_norm', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, 'SumToOneNormLayer must have 1 input') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('cos_vm') class CosSimVecMatLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - cos_scale=1.0, - device=None): + def __init__(self, name, size, inputs, cos_scale=1.0, device=None): super(CosSimVecMatLayer, self).__init__( - name, 'cos_vm', size, inputs=inputs, device=device) + name, 'cos_vm', size, inputs=inputs, device=device) self.config.cos_scale = cos_scale - config_assert(len(self.inputs) == 2, - 'CosSimVecMatLayer must have 2 inputs') + config_assert( + len(self.inputs) == 2, 'CosSimVecMatLayer must have 2 inputs') config_assert( size * self.get_input_layer(0).size == self.get_input_layer(1).size, 'Wrong input size for CosSimVecMatLayer') + @config_layer('sampling_id') class SamplingIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(SamplingIdLayer, self).__init__( name, 'sampling_id', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, 'SamplingIdLayer must have 1 input') + config_assert( + len(self.inputs) == 1, 'SamplingIdLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2603,33 +2553,33 @@ class SamplingIdLayer(LayerBase): # 'squarerootn': sum each sample, but divide by sqrt(sample_num). @config_layer('average') class AverageLayer(LayerBase): - def __init__( - self, - name, - inputs, - average_strategy='average', - trans_type='non-seq', - active_type='linear', - device=None, - bias=False): - super(AverageLayer, self).__init__(name, 'average', 0, inputs=inputs, - device=device, active_type=active_type) + def __init__(self, + name, + inputs, + average_strategy='average', + trans_type='non-seq', + active_type='linear', + device=None, + bias=False): + super(AverageLayer, self).__init__( + name, + 'average', + 0, + inputs=inputs, + device=device, + active_type=active_type) self.config.average_strategy = average_strategy - self.config.trans_type = trans_type + self.config.trans_type = trans_type config_assert(len(inputs) == 1, 'AverageLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('cos') class CosSimLayer(LayerBase): - def __init__( - self, - name, - inputs, - cos_scale=5, - device=None): + def __init__(self, name, inputs, cos_scale=5, device=None): super(CosSimLayer, self).__init__( name, 'cos', 1, inputs=inputs, device=device) config_assert(len(self.inputs) == 2, 'CosSimLayer must have 2 inputs') @@ -2641,18 +2591,13 @@ class CosSimLayer(LayerBase): @config_layer('tensor') class TensorLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None, - bias=True, - **xargs): - super(TensorLayer, self).__init__(name, 'tensor', size, inputs=inputs, device=device, **xargs) + def __init__(self, name, size, inputs, device=None, bias=True, **xargs): + super(TensorLayer, self).__init__( + name, 'tensor', size, inputs=inputs, device=device, **xargs) config_assert(len(self.inputs) == 2, 'TensorLayer must have 2 inputs') config_assert(size > 0, 'size must be positive') - config_assert(inputs[1].parameter_name == None, 'second parameter should be None.') + config_assert(inputs[1].parameter_name == None, + 'second parameter should be None.') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) psize = size * input_layer0.size * input_layer1.size @@ -2663,14 +2608,13 @@ class TensorLayer(LayerBase): @config_layer('mixed') class MixedLayer(LayerBase): - def __init__( - self, - name, - inputs, - size=0, - bias=True, - error_clipping_threshold=None, - **xargs): + def __init__(self, + name, + inputs, + size=0, + bias=True, + error_clipping_threshold=None, + **xargs): config_assert(inputs, 'inputs cannot be empty') super(MixedLayer, self).__init__( name, 'mixed', size, inputs=inputs, **xargs) @@ -2695,24 +2639,28 @@ class MixedLayer(LayerBase): else: sz = operator.calc_output_size(operator_conf.input_sizes) if sz != 0: - config_assert(sz == self.config.size, - "different inputs have different size: %s vs. %s" % - (sz, self.config.size)) + config_assert( + sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] if input_index not in operator_input_index: - config_assert(isinstance(input, Projection), "input should be projection or operation") + config_assert( + isinstance(input, Projection), + "input should be projection or operation") if self.config.size == 0 and isinstance(input, Projection): size = input.calc_output_size(input_layer) if size != 0: self.set_layer_size(size) elif isinstance(input, Projection): - sz = input.calc_output_size(input_layer) - if sz != 0: - config_assert(sz == self.config.size, - "different inputs have different size: %s vs. %s" % - (sz, self.config.size)) + sz = input.calc_output_size(input_layer) + if sz != 0: + config_assert( + sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) config_assert(size != 0, "size is not set") for input_index in xrange(len(self.inputs)): @@ -2724,7 +2672,8 @@ class MixedLayer(LayerBase): input_config = self.config.inputs[input_index] input_config.proj_conf.CopyFrom(input.proj_conf) - input_config.proj_conf.name = gen_parameter_name(name, input_index) + input_config.proj_conf.name = gen_parameter_name(name, + input_index) psize = input.calc_parameter_size(input_layer.size, size) dims = input.calc_parameter_dims(input_layer.size, size) self.create_input_parameter(input_index, psize, dims) @@ -2750,21 +2699,16 @@ class MixedLayer(LayerBase): if error_clipping_threshold is not None: self.config.error_clipping_threshold = error_clipping_threshold + # like MixedLayer, but no bias parameter @config_func -def ExpressionLayer(name, - inputs, - **xargs): +def ExpressionLayer(name, inputs, **xargs): MixedLayer(name, inputs, bias=False, **xargs) + @config_layer('concat') class ConcatenateLayer(LayerBase): - def __init__( - self, - name, - inputs, - bias=False, - **xargs): + def __init__(self, name, inputs, bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') config_assert(not bias, 'ConcatenateLayer cannot support bias.') super(ConcatenateLayer, self).__init__( @@ -2773,30 +2717,27 @@ class ConcatenateLayer(LayerBase): for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] - if self.config.size == 0: + if self.config.size == 0: size += input_layer.size self.set_layer_size(size) + # like concat layer, but each input layer was processed by a Projection. @config_layer('concat2') class ConcatenateLayer2(LayerBase): - def __init__( - self, - name, - inputs, - bias=False, - **xargs): + def __init__(self, name, inputs, bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') super(ConcatenateLayer2, self).__init__( name, 'concat2', 0, inputs=inputs, **xargs) if isinstance(self.inputs[0], ConvProjection): - for input_index in xrange(len(self.inputs) - 1): - input = self.inputs[input_index + 1] - config_assert(isinstance(input, ConvProjection), - "The first input of ConcatenateLayer2 is ConvProjection, " - "the other inputs should also be ConvProjection.") + for input_index in xrange(len(self.inputs) - 1): + input = self.inputs[input_index + 1] + config_assert( + isinstance(input, ConvProjection), + "The first input of ConcatenateLayer2 is ConvProjection, " + "the other inputs should also be ConvProjection.") size = 0 for input_index in xrange(len(self.inputs)): @@ -2818,9 +2759,9 @@ class ConcatenateLayer2(LayerBase): input_config.proj_conf.CopyFrom(input.proj_conf) input_config.proj_conf.name = gen_parameter_name(name, input_index) psize = input.calc_parameter_size(input.proj_conf.input_size, - input.proj_conf.output_size) + input.proj_conf.output_size) dims = input.calc_parameter_dims(input.proj_conf.input_size, - input.proj_conf.output_size) + input.proj_conf.output_size) self.create_input_parameter(input_index, psize, dims) psize = self.config.size @@ -2834,16 +2775,12 @@ class ConcatenateLayer2(LayerBase): self.config.bias_size = psize self.create_bias_parameter(bias, psize) + @config_layer('recurrent') class RecurrentLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - bias=True, - **xargs): - super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, **xargs) + def __init__(self, name, inputs, reversed=False, bias=True, **xargs): + super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, ** + xargs) config_assert(len(self.inputs) == 1, 'RecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) size = input_layer.size @@ -2853,17 +2790,17 @@ class RecurrentLayer(LayerBase): self.create_input_parameter(0, size * size, dims) self.create_bias_parameter(bias, self.config.size) + @config_layer('lstmemory') class LstmLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): + def __init__(self, + name, + inputs, + reversed=False, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): super(LstmLayer, self).__init__(name, 'lstmemory', 0, inputs, **xargs) config_assert(len(self.inputs) == 1, 'LstmLayer must have 1 input') input_layer = self.get_input_layer(0) @@ -2872,117 +2809,126 @@ class LstmLayer(LayerBase): size = input_layer.size / 4 self.set_layer_size(size) self.config.reversed = reversed - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type self.create_input_parameter(0, size * size * 4, [size, size, 4]) #bias includes 3 kinds of peephole, 4 + 3 = 7 self.create_bias_parameter(bias, size * 7) + @config_layer('lstm_step') class LstmStepLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): - super(LstmStepLayer, self).__init__(name, 'lstm_step', - size, inputs, **xargs) + def __init__(self, + name, + size, + inputs, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): + super(LstmStepLayer, self).__init__(name, 'lstm_step', size, inputs, + **xargs) config_assert(len(inputs) == 2, 'LstmStepLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) - config_assert(input_layer0.size == 4 * size, 'input_layer0.size != 4 * layer.size') - config_assert(input_layer1.size == size, 'input_layer1.size != layer.size') - self.config.active_gate_type = active_gate_type + config_assert(input_layer0.size == 4 * size, + 'input_layer0.size != 4 * layer.size') + config_assert(input_layer1.size == size, + 'input_layer1.size != layer.size') + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type self.create_bias_parameter(bias, size * 3) + # get the specific output from the input layer. @config_layer('get_output') class GetOutputLayer(LayerBase): - def __init__( - self, - name, - size, - inputs): - super(GetOutputLayer, self).__init__(name, 'get_output' , size, inputs) - config_assert(len(self.inputs) == 1, 'GetOutputLayer must have 1 inputs') + def __init__(self, name, size, inputs): + super(GetOutputLayer, self).__init__(name, 'get_output', size, inputs) + config_assert( + len(self.inputs) == 1, 'GetOutputLayer must have 1 inputs') inputs = self.inputs[0] config_assert(inputs.input_layer_argument, 'input_layer_argument cannot be empty') + @config_layer('mdlstmemory') class MDLstmLayer(LayerBase): - def __init__( - self, - name, - inputs, - directions=True, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): - super(MDLstmLayer, self).__init__(name, 'mdlstmemory', 0, inputs, **xargs) + def __init__(self, + name, + inputs, + directions=True, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): + super(MDLstmLayer, self).__init__(name, 'mdlstmemory', 0, inputs, ** + xargs) config_assert(len(self.inputs) == 1, 'MDLstmLayer must have 1 input') input_layer = self.get_input_layer(0) dim_num = len(directions) #check input_layer.size is divided by (3+dim_num) - config_assert(input_layer.size % (3+dim_num) == 0, "size % (dim_num) should be 0!") - size = input_layer.size / (3+dim_num) + config_assert(input_layer.size % + (3 + dim_num) == 0, "size % (dim_num) should be 0!") + size = input_layer.size / (3 + dim_num) self.set_layer_size(size) - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type for i in xrange(len(directions)): self.config.directions.append(int(directions[i])) - self.create_input_parameter(0, size * size * (3+dim_num), [size, size, 3+dim_num]) + self.create_input_parameter(0, size * size * + (3 + dim_num), [size, size, 3 + dim_num]) #bias includes 3 kinds of peephole, 3+dim_num+2+dim_num - self.create_bias_parameter(bias, size * (5+2*dim_num)) + self.create_bias_parameter(bias, size * (5 + 2 * dim_num)) + @config_layer('gated_recurrent') class GatedRecurrentLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - active_gate_type="sigmoid", - bias=True, - **xargs): - super(GatedRecurrentLayer, self).__init__(name, 'gated_recurrent', 0, inputs, **xargs) - config_assert(len(self.inputs) == 1, 'GatedRecurrentLayer must have 1 input') + def __init__(self, + name, + inputs, + reversed=False, + active_gate_type="sigmoid", + bias=True, + **xargs): + super(GatedRecurrentLayer, self).__init__(name, 'gated_recurrent', 0, + inputs, **xargs) + config_assert( + len(self.inputs) == 1, 'GatedRecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) #check input_layer.size is divided by 3 config_assert(input_layer.size % 3 == 0, "size % 3 should be 0!") size = input_layer.size / 3 self.set_layer_size(size) self.config.reversed = reversed - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.create_input_parameter(0, size * size * 3, [size, size * 3]) self.create_bias_parameter(bias, size * 3) + @config_layer('gru_step') class GruStepLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - active_gate_type="sigmoid", - bias=True, - **xargs): - super(GruStepLayer, self).__init__(name, 'gru_step', size, inputs, **xargs) + def __init__(self, + name, + size, + inputs, + active_gate_type="sigmoid", + bias=True, + **xargs): + super(GruStepLayer, self).__init__(name, 'gru_step', size, inputs, ** + xargs) config_assert(len(self.inputs) == 2, 'GruStepLayer must have 2 input') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) - config_assert(input_layer0.size == 3 * size, 'input_layer0.size != 3 * layer.size') - config_assert(input_layer1.size == size, 'input_layer1.size != layer.size') - self.config.active_gate_type = active_gate_type + config_assert(input_layer0.size == 3 * size, + 'input_layer0.size != 3 * layer.size') + config_assert(input_layer1.size == size, + 'input_layer1.size != layer.size') + self.config.active_gate_type = active_gate_type self.create_input_parameter(0, size * size * 3, [size, size * 3]) self.create_bias_parameter(bias, size * 3) + ''' A layer for calculating the cost of sequential conditional random field model. Example: CRFLayer(name="crf_cost", size=label_num, @@ -2990,20 +2936,18 @@ class GruStepLayer(LayerBase): where "weight" is optional, one weight for each sequence @param coeff: weight of the layer ''' + + @config_layer('crf') class CRFLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - coeff=1.0, - device=None): + def __init__(self, name, size, inputs, coeff=1.0, device=None): super(CRFLayer, self).__init__(name, 'crf', size, inputs, device=device) - config_assert(2 <= len(self.inputs) <= 3, 'CRFLayer must have 2 or 3 inputs') + config_assert(2 <= len(self.inputs) <= 3, + 'CRFLayer must have 2 or 3 inputs') self.create_input_parameter(0, size * (size + 2), [size, size + 2]) self.config.coeff = coeff + ''' A layer for calculating the decoding sequence of sequential conditional random field model. @@ -3012,14 +2956,11 @@ class CRFLayer(LayerBase): this layer will also calculate error, output_.value[i] is 1 for incorrect decoding or 0 for correct decoding ''' + + @config_layer('crf_decoding') class CRFDecodingLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): + def __init__(self, name, size, inputs, device=None): super(CRFDecodingLayer, self).__init__( name, 'crf_decoding', size, inputs, device=device) config_assert( @@ -3027,47 +2968,35 @@ class CRFDecodingLayer(LayerBase): 'CRFDecodingLayer cannot have more than 2 inputs') self.create_input_parameter(0, size * (size + 2), [size, size + 2]) + @config_layer('ctc') class CTCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - norm_by_times = False, - device=None): + def __init__(self, name, size, inputs, norm_by_times=False, device=None): super(CTCLayer, self).__init__(name, 'ctc', size, inputs, device=device) self.config.norm_by_times = norm_by_times config_assert(len(self.inputs) == 2, 'CTCLayer must have 2 inputs') + @config_layer('recurrent_layer_group') class RecurrentLayerGroup(LayerBase): - def __init__( - self, - name, - device=None): + def __init__(self, name, device=None): super(RecurrentLayerGroup, self).__init__( name, 'recurrent_layer_group', 0, inputs=[], device=device) # Deprecated, use a new layer specific class instead @config_func -def Layer( - name, - type, - **xargs): +def Layer(name, type, **xargs): layers = {} layers.update(g_cost_map) layers.update(g_layer_type_map) layer_func = layers.get(type) - config_assert(layer_func, - "layer type '%s' not supported." % type) + config_assert(layer_func, "layer type '%s' not supported." % type) return layer_func(name, **xargs) + @config_func -def ParameterHook( - type, - **kwargs): +def ParameterHook(type, **kwargs): if type == 'pruning': mask_filename = kwargs.get('mask_filename', None) assert mask_filename is not None @@ -3080,30 +3009,28 @@ def ParameterHook( @config_func -def Parameter( - name, - size, - device, - dims, - learning_rate=None, - momentum=None, - decay_rate=None, - decay_rate_l1=None, - initial_mean=None, - initial_std=None, - initial_strategy=None, - initial_smart=None, - num_batches_regularization=None, - sparse_remote_update=None, - sparse_update=None, - gradient_clipping_threshold=None, - sparse=None, - format=None, - need_compact=None, - is_static=None, - is_shared=None, - update_hooks=None - ): +def Parameter(name, + size, + device, + dims, + learning_rate=None, + momentum=None, + decay_rate=None, + decay_rate_l1=None, + initial_mean=None, + initial_std=None, + initial_strategy=None, + initial_smart=None, + num_batches_regularization=None, + sparse_remote_update=None, + sparse_update=None, + gradient_clipping_threshold=None, + sparse=None, + format=None, + need_compact=None, + is_static=None, + is_shared=None, + update_hooks=None): config_assert(name not in g_parameter_map, 'Duplicated parameter name: ' + name) @@ -3134,8 +3061,8 @@ def Parameter( para.initial_std = default(initial_std, g_default_initial_std) para.initial_mean = default(initial_mean, g_default_initial_mean) - num_batches_regularization = default( - num_batches_regularization, g_default_num_batches_regularization) + num_batches_regularization = default(num_batches_regularization, + g_default_num_batches_regularization) if num_batches_regularization is not None: para.num_batches_regularization = int(num_batches_regularization) @@ -3145,18 +3072,21 @@ def Parameter( g_config.opt_config.use_sparse_remote_updater = True if sparse_update is not None: para.sparse_update = sparse_update - gradient_clipping_threshold = default( - gradient_clipping_threshold, g_default_gradient_clipping_threshold) + gradient_clipping_threshold = default(gradient_clipping_threshold, + g_default_gradient_clipping_threshold) if gradient_clipping_threshold is not None: para.gradient_clipping_threshold = gradient_clipping_threshold - para.initial_strategy = default(initial_strategy, g_default_initial_strategy) + para.initial_strategy = default(initial_strategy, + g_default_initial_strategy) para.initial_smart = default(initial_smart, g_default_initial_smart) if para.initial_smart: para.initial_mean = 0. if len(para.dims) != 0: para.initial_std = 1. / math.sqrt(para.dims[0]) else: - print("Use initial_smart, but dims not set. Initial_smart may not be used in this layer") + print( + "Use initial_smart, but dims not set. Initial_smart may not be used in this layer" + ) traceback.print_exc() para.initial_std = 1. / math.sqrt(para.size) if g_default_compact_func is not None: @@ -3195,64 +3125,78 @@ def default_initial_std(val): global g_default_initial_std g_default_initial_std = val + @config_func def default_initial_mean(val): global g_default_initial_mean g_default_initial_mean = val + @config_func def default_initial_strategy(val): global g_default_initial_strategy g_default_initial_strategy = val + @config_func def default_initial_smart(val): global g_default_initial_smart g_default_initial_smart = val + @config_func def default_momentum(val): global g_default_momentum g_default_momentum = val + @config_func def default_decay_rate(val): global g_default_decay_rate g_default_decay_rate = val + @config_func def default_num_batches_regularization(val): global g_default_num_batches_regularization g_default_num_batches_regularization = val + @config_func def default_gradient_clipping_threshold(val): global g_default_gradient_clipping_threshold g_default_gradient_clipping_threshold = val + @config_func def default_device(val): global g_default_device g_default_device = val + @config_func def default_update_hooks(val): global g_default_update_hooks g_default_update_hooks = val + @config_func def default_compact_func(val): global g_default_compact_func g_default_compact_func = val + def make_importer(config_dir, config_args): def Import(config_file, local_args={}): if not config_file.startswith('/'): config_file = config_dir + '/' + config_file g_config.config_files.append(config_file) - execfile(config_file, make_config_environment(config_file, config_args), local_args) + execfile(config_file, + make_config_environment(config_file, config_args), local_args) + return Import + settings = dict( batch_size=None, mini_batch_size=None, @@ -3281,26 +3225,24 @@ settings = dict( ada_rou=0.95, delta_add_rate=1.0, shrink_parameter_value=0, - adam_beta1 = 0.9, - adam_beta2 = 0.999, - adam_epsilon = 1e-8, -) + adam_beta1=0.9, + adam_beta2=0.999, + adam_epsilon=1e-8, ) -settings_deprecated = dict( - usage_ratio=1., -) +settings_deprecated = dict(usage_ratio=1., ) trainer_settings = dict( save_dir="./output/model", init_model_path=None, - start_pass=0, -) + start_pass=0, ) + @config_func def Settings(**args): for k, v in args.iteritems(): if k == "usage_ratio": - logger.warning("Deprecated: define usage_ratio in DataConfig instead") + logger.warning( + "Deprecated: define usage_ratio in DataConfig instead") if g_config.HasField("data_config"): g_config.data_config.__setattr__(k, v) settings_deprecated[k] = v @@ -3312,10 +3254,12 @@ def Settings(**args): else: logger.fatal('Unkown setting: %s' % k) + @config_func def cluster_config(**args): pass + @config_func def EnableSubmodelSuffix(flag=True): """ @@ -3325,10 +3269,12 @@ def EnableSubmodelSuffix(flag=True): global g_add_submodel_suffix g_add_submodel_suffix = flag + def make_config_environment(config_file, config_args): def make_setter(k): def setter(v): logger.fatal("Obsolete: use Settings(%s=%s, ...) instead" % (k, v)) + return setter funcs = {} @@ -3344,13 +3290,13 @@ def make_config_environment(config_file, config_args): funcs.update( Import=make_importer(config_dir, config_args), - get_config_arg=make_get_config_arg(config_args), - ) + get_config_arg=make_get_config_arg(config_args), ) funcs.update(g_extended_config_funcs) return funcs + def make_get_config_arg(config_args): def get_config_arg(name, type, default=None): if type == bool: @@ -3367,6 +3313,7 @@ def make_get_config_arg(config_args): return get_config_arg + def importlib(name): __import__(name) return sys.modules[name] @@ -3379,10 +3326,12 @@ def find_caller(): return s[0], s[1], s[2] return "(unknown file)", 0, "(unknown function)" + def my_fatal(s): logger.critical(s) raise Exception() + def parse_config(config_file, config_arg_str): ''' @param config_arg_str: a string of the form var1=val1,var2=val2. It will be @@ -3420,7 +3369,7 @@ def parse_config(config_file, config_arg_str): for k, v in settings.iteritems(): if v is None: continue - g_config.opt_config.__setattr__(k, v); + g_config.opt_config.__setattr__(k, v) for k, v in trainer_settings.iteritems(): if v is None: @@ -3447,6 +3396,7 @@ def parse_config_and_serialize(config_file, config_arg_str): traceback.print_exc() raise + if __name__ == '__main__': try: config = parse_config(sys.argv[1], '') diff --git a/python/paddle/trainer/config_parser_extension.py b/python/paddle/trainer/config_parser_extension.py index 3445076274..ba4c79efdc 100644 --- a/python/paddle/trainer/config_parser_extension.py +++ b/python/paddle/trainer/config_parser_extension.py @@ -17,11 +17,10 @@ from paddle.proto.DataConfig_pb2 import DataConfig g_config = None -def SimpleData( - files=None, - feat_dim=None, - context_len=None, - buffer_capacity=None): +def SimpleData(files=None, + feat_dim=None, + context_len=None, + buffer_capacity=None): data_config = DataConfig() data_config.type = 'simple' @@ -33,6 +32,7 @@ def SimpleData( data_config.buffer_capacity = buffer_capacity return data_config + def get_config_funcs(trainer_config): global g_config g_config = trainer_config diff --git a/python/paddle/trainer/recurrent_units.py b/python/paddle/trainer/recurrent_units.py index 7d51de78b0..a80ad13d1e 100644 --- a/python/paddle/trainer/recurrent_units.py +++ b/python/paddle/trainer/recurrent_units.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - + # recurrent_units.py # Version 2.0 # @@ -22,161 +22,175 @@ from paddle.trainer.config_parser import * + # long short term memory, can be used in recurrent machine # *inputs* must be a list of Projections, for example: # inputs = [FullMatrixProjection("input_layer_name")], # *para_prefix* defines parameter names, if the *para_prefix* of # two LstmRecurrentUnit is same, they share same parameters # *out_memory* can be defined outside if it's used outside -def LstmRecurrentUnit(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): +def LstmRecurrentUnit(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): - if para_prefix is None: + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) + + state_memory = Memory(name=name + "_" + "state", size=size) - state_memory = Memory(name = name + "_" + "state", size = size) - Layer( - name = name + "_" + "input_recurrent", - type = "mixed", - size = size * 4, #(input_s, input_gate, forget_gate, output_gate) - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, - parameter_name = para_prefix + "_input_recurrent.b"), - inputs = inputs + [ - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_input_recurrent.w"), - ], - ) + name=name + "_" + "input_recurrent", + type="mixed", + size=size * 4, #(input_s, input_gate, forget_gate, output_gate) + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_input_recurrent.b"), + inputs=inputs + [ + FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_input_recurrent.w"), + ], ) LstmStepLayer( - name = name, - size = size, - bias = Bias(parameter_name = para_prefix + "_check.b"), - inputs = [name + "_" + "input_recurrent", state_memory], - active_type = active_type, - active_gate_type = gate_active_type, - active_state_type = state_active_type, - ) + name=name, + size=size, + bias=Bias(parameter_name=para_prefix + "_check.b"), + inputs=[name + "_" + "input_recurrent", state_memory], + active_type=active_type, + active_gate_type=gate_active_type, + active_state_type=state_active_type, ) GetOutputLayer( - name = name + "_" + "state", - size = size, - inputs = Input(name, input_layer_argument = "state"), - ) - -def LstmRecurrentUnitNaive(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - - if para_prefix is None: + name=name + "_" + "state", + size=size, + inputs=Input( + name, input_layer_argument="state"), ) + + +def LstmRecurrentUnitNaive(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) + + state_memory = Memory(name=name + "_" + "state", size=size) - state_memory = Memory(name = name + "_" + "state", size = size) - Layer( - name = name + "_" + "input_recurrent", - type = "mixed", - size = size * 4, #(input_s, input_gate, forget_gate, output_gate) - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, - parameter_name = para_prefix + "_input_recurrent.b"), - inputs = inputs + [ - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_input_recurrent.w"), - ], - ) + name=name + "_" + "input_recurrent", + type="mixed", + size=size * 4, #(input_s, input_gate, forget_gate, output_gate) + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_input_recurrent.b"), + inputs=inputs + [ + FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_input_recurrent.w"), + ], ) ExpressionLayer( - name = name + "_" + "input_s", - size = size, - active_type = active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=0)], - ) + name=name + "_" + "input_s", + size=size, + active_type=active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=0) + ], ) ExpressionLayer( - name = name + "_" + "input_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size), - DotMulProjection(state_memory, - parameter_name = para_prefix + "_input_check.w")], - ) + name=name + "_" + "input_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size), DotMulProjection( + state_memory, parameter_name=para_prefix + "_input_check.w") + ], ) ExpressionLayer( - name = name + "_" + "forget_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size*2), - DotMulProjection(state_memory, - parameter_name = para_prefix + "_forget_check.w")], - ) + name=name + "_" + "forget_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size * 2), + DotMulProjection( + state_memory, parameter_name=para_prefix + "_forget_check.w") + ], ) ExpressionLayer( - name = name + "_" + "state", - inputs = [DotMulOperator([name + "_" + "input_s", - name + "_" + "input_gate"]), - DotMulOperator([state_memory, - name + "_" + "forget_gate"]), - ], - ) + name=name + "_" + "state", + inputs=[ + DotMulOperator([name + "_" + "input_s", name + "_" + "input_gate"]), + DotMulOperator([state_memory, name + "_" + "forget_gate"]), + ], ) ExpressionLayer( - name = name + "_" + "output_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size*3), - DotMulProjection(name + "_" + "state", - parameter_name = para_prefix + "_output_check.w")], - ) + name=name + "_" + "output_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size * 3), + DotMulProjection( + name + "_" + "state", + parameter_name=para_prefix + "_output_check.w") + ], ) ExpressionLayer( - name = name + "_" + "state_atv", - active_type = state_active_type, - inputs = IdentityProjection(name + "_" + "state"), - ) + name=name + "_" + "state_atv", + active_type=state_active_type, + inputs=IdentityProjection(name + "_" + "state"), ) ExpressionLayer( - name = name, - inputs = DotMulOperator([name + "_" + "state_atv", - name + "_" + "output_gate"]), - ) + name=name, + inputs=DotMulOperator( + [name + "_" + "state_atv", name + "_" + "output_gate"]), ) + # like LstmRecurrentUnit, but it's a layer group. # it is equivalent to LstmLayer -def LstmRecurrentLayerGroup(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - seq_reversed = False): +def LstmRecurrentLayerGroup(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + seq_reversed=False): input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 4, - active_type = "", - bias = False, - inputs = inputs, - ) - - RecurrentLayerGroupBegin(name + "_layer_group", - in_links = [input_layer_name], - out_links = [name], - seq_reversed = seq_reversed) + name=input_layer_name, + type="mixed", + size=size * 4, + active_type="", + bias=False, + inputs=inputs, ) + + RecurrentLayerGroupBegin( + name + "_layer_group", + in_links=[input_layer_name], + out_links=[name], + seq_reversed=seq_reversed) LstmRecurrentUnit( - name = name, - size = size, - active_type = active_type, - state_active_type = state_active_type, - gate_active_type = gate_active_type, - inputs = [IdentityProjection(input_layer_name)], - para_prefix = para_prefix, - error_clipping_threshold = error_clipping_threshold, - ) + name=name, + size=size, + active_type=active_type, + state_active_type=state_active_type, + gate_active_type=gate_active_type, + inputs=[IdentityProjection(input_layer_name)], + para_prefix=para_prefix, + error_clipping_threshold=error_clipping_threshold, ) RecurrentLayerGroupEnd(name + "_layer_group") - # gated recurrent unit, can be used in recurrent machine # *inputs* should be a list of Projections, for example: # inputs = [FullMatrixProjection("input_layer_name")], @@ -184,142 +198,157 @@ def LstmRecurrentLayerGroup(name, size, # two GatedRecurrentUnit is same, they share same parameters # *out_memory* can be defined outside if it's used outside -def GatedRecurrentUnit(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup + +def GatedRecurrentUnit(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup input_layer_name = inputs else: input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - if para_prefix is None: + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) GruStepLayer( - name = name, - size = size, - bias = Bias(parameter_name = para_prefix + "_gate.b"), - inputs = [input_layer_name, - Input(out_memory, parameter_name = para_prefix + "_gate.w")], - active_type = active_type, - active_gate_type = gate_active_type, - ) - -def GatedRecurrentUnitNaive(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - - if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup + name=name, + size=size, + bias=Bias(parameter_name=para_prefix + "_gate.b"), + inputs=[ + input_layer_name, Input( + out_memory, parameter_name=para_prefix + "_gate.w") + ], + active_type=active_type, + active_gate_type=gate_active_type, ) + + +def GatedRecurrentUnitNaive(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + + if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup input_layer_name = inputs else: input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - if para_prefix is None: + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) Layer( - name = name + "_" + "update_gate", - type = "mixed", - size = size, - active_type = gate_active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_update_gate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=0), - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_update_gate.w")], - ) + name=name + "_" + "update_gate", + type="mixed", + size=size, + active_type=gate_active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_update_gate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=0), FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_update_gate.w") + ], ) Layer( - name = name + "_" + "reset_gate", - type = "mixed", - size = size, - active_type = gate_active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_reset_gate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=size), - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_reset_gate.w")], - ) + name=name + "_" + "reset_gate", + type="mixed", + size=size, + active_type=gate_active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_reset_gate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=size), FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_reset_gate.w") + ], ) ExpressionLayer( - name = name + "_" + "reset_output", - inputs = DotMulOperator([out_memory, name + "_" + "reset_gate"]), - ) + name=name + "_" + "reset_output", + inputs=DotMulOperator([out_memory, name + "_" + "reset_gate"]), ) Layer( - name = name + "_" + "output_candidate", - type = "mixed", - size = size, - active_type = active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_output_candidate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=size*2), - FullMatrixProjection(name + "_" + "reset_output", - parameter_name = para_prefix + "_output_candidate.w")], - ) - ExpressionLayer( #element-wise interpolation - name = name, - inputs = [IdentityProjection(out_memory), - DotMulOperator([out_memory, - name + "_" + "update_gate"], scale=-1.0), - DotMulOperator([name + "_" + "output_candidate", - name + "_" + "update_gate"]), - ], - ) + name=name + "_" + "output_candidate", + type="mixed", + size=size, + active_type=active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_output_candidate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=size * 2), FullMatrixProjection( + name + "_" + "reset_output", + parameter_name=para_prefix + "_output_candidate.w") + ], ) + ExpressionLayer( #element-wise interpolation + name=name, + inputs=[ + IdentityProjection(out_memory), + DotMulOperator( + [out_memory, name + "_" + "update_gate"], scale=-1.0), + DotMulOperator( + [name + "_" + "output_candidate", name + "_" + "update_gate"]), + ], ) + # like GatedRecurrentUnit, but it's a layer group. # it is equivalent to GatedRecurrentLayer. -def GatedRecurrentLayerGroup(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - seq_reversed = False): +def GatedRecurrentLayerGroup(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + seq_reversed=False): input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - RecurrentLayerGroupBegin(name + "_layer_group", - in_links = [input_layer_name], - out_links = [name], - seq_reversed = seq_reversed) + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + RecurrentLayerGroupBegin( + name + "_layer_group", + in_links=[input_layer_name], + out_links=[name], + seq_reversed=seq_reversed) GatedRecurrentUnit( - name = name, - size = size, - active_type = active_type, - gate_active_type = gate_active_type, - inputs = input_layer_name, #transform outside - para_prefix = para_prefix, - error_clipping_threshold = error_clipping_threshold, - ) + name=name, + size=size, + active_type=active_type, + gate_active_type=gate_active_type, + inputs=input_layer_name, #transform outside + para_prefix=para_prefix, + error_clipping_threshold=error_clipping_threshold, ) RecurrentLayerGroupEnd(name + "_layer_group") - diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index 2202d0bf96..6261934e1b 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ["TanhActivation", "SigmoidActivation", - "SoftmaxActivation", "IdentityActivation", "LinearActivation", - 'SequenceSoftmaxActivation', 'ExpActivation', - "ReluActivation", "BReluActivation", "SoftReluActivation", - "STanhActivation", - "AbsActivation", "SquareActivation", - "BaseActivation"] +__all__ = [ + "TanhActivation", "SigmoidActivation", "SoftmaxActivation", + "IdentityActivation", "LinearActivation", 'SequenceSoftmaxActivation', + 'ExpActivation', "ReluActivation", "BReluActivation", "SoftReluActivation", + "STanhActivation", "AbsActivation", "SquareActivation", "BaseActivation" +] class BaseActivation(object): @@ -51,7 +50,8 @@ class TanhActivation(BaseActivation): f(z)=tanh(z)=\\frac{e^z-e^{-z}}{e^z+e^{-z}} """ - def __init__(self): BaseActivation.__init__(self, 'tanh', True) + def __init__(self): + BaseActivation.__init__(self, 'tanh', True) class SigmoidActivation(BaseActivation): @@ -63,7 +63,8 @@ class SigmoidActivation(BaseActivation): f(z) = \\frac{1}{1+exp(-z)} """ - def __init__(self): BaseActivation.__init__(self, 'sigmoid', True) + def __init__(self): + BaseActivation.__init__(self, 'sigmoid', True) class SoftmaxActivation(BaseActivation): @@ -104,7 +105,8 @@ class IdentityActivation(BaseActivation): Just do nothing for output both forward/backward. """ - def __init__(self): BaseActivation.__init__(self, '', False) + def __init__(self): + BaseActivation.__init__(self, '', False) LinearActivation = IdentityActivation @@ -124,7 +126,8 @@ class ReluActivation(BaseActivation): 0 &\\quad\\mathrm{otherwize} """ - def __init__(self): BaseActivation.__init__(self, 'relu', True) + def __init__(self): + BaseActivation.__init__(self, 'relu', True) class BReluActivation(BaseActivation): @@ -141,7 +144,8 @@ class BReluActivation(BaseActivation): 0 &\\quad \\mathrm{otherwise} """ - def __init__(self): BaseActivation.__init__(self, 'brelu', False) + def __init__(self): + BaseActivation.__init__(self, 'brelu', False) class SoftReluActivation(BaseActivation): @@ -149,7 +153,9 @@ class SoftReluActivation(BaseActivation): SoftRelu Activation. """ - def __init__(self): BaseActivation.__init__(self, 'softrelu', False) + def __init__(self): + BaseActivation.__init__(self, 'softrelu', False) + class STanhActivation(BaseActivation): """ @@ -160,7 +166,8 @@ class STanhActivation(BaseActivation): f(z) = 1.7159 * tanh(2/3*z) """ - def __init__(self): BaseActivation.__init__(self, 'stanh', False) + def __init__(self): + BaseActivation.__init__(self, 'stanh', False) class AbsActivation(BaseActivation): @@ -178,7 +185,8 @@ class AbsActivation(BaseActivation): 0 &\\quad if \\quad z = 0 """ - def __init__(self): BaseActivation.__init__(self, 'abs', False) + def __init__(self): + BaseActivation.__init__(self, 'abs', False) class SquareActivation(BaseActivation): @@ -189,7 +197,9 @@ class SquareActivation(BaseActivation): f(z) = z^2. """ - def __init__(self): BaseActivation.__init__(self, 'square', False) + def __init__(self): + BaseActivation.__init__(self, 'square', False) + class ExpActivation(BaseActivation): """ @@ -198,7 +208,10 @@ class ExpActivation(BaseActivation): .. math:: f(z) = e^z. """ - def __init__(self): BaseActivation.__init__(self, 'exponential', False) + + def __init__(self): + BaseActivation.__init__(self, 'exponential', False) + class LogActivation(BaseActivation): """ @@ -207,4 +220,6 @@ class LogActivation(BaseActivation): .. math:: f(z) = log(z) """ - def __init__(self): BaseActivation.__init__(self, 'log', False) + + def __init__(self): + BaseActivation.__init__(self, 'log', False) diff --git a/python/paddle/trainer_config_helpers/attrs.py b/python/paddle/trainer_config_helpers/attrs.py index d263441247..54169f382f 100644 --- a/python/paddle/trainer_config_helpers/attrs.py +++ b/python/paddle/trainer_config_helpers/attrs.py @@ -13,8 +13,9 @@ # limitations under the License. from paddle.trainer.config_parser import * -__all__ = ['ParamAttr', 'ExtraAttr', 'ParameterAttribute', - 'ExtraLayerAttribute'] +__all__ = [ + 'ParamAttr', 'ExtraAttr', 'ParameterAttribute', 'ExtraLayerAttribute' +] def convert_and_compare(x, Type): @@ -25,7 +26,8 @@ def convert_and_compare(x, Type): :param Type: target type to check x over """ - return type(x)(Type(x))==x + return type(x)(Type(x)) == x + def is_compatible_with(x, Type): """ @@ -38,9 +40,9 @@ def is_compatible_with(x, Type): return True try: if float == Type or int == Type: - # avoid those types that can be converted to float/int but not very - # meaningful and could potentially lead to error - # i.e., str and bool typed value should not be used for initializing float/int variable + # avoid those types that can be converted to float/int but not very + # meaningful and could potentially lead to error + # i.e., str and bool typed value should not be used for initializing float/int variable if not isinstance(x, str) and not isinstance(x, bool): return convert_and_compare(x, Type) elif bool == Type: @@ -91,9 +93,17 @@ class ParameterAttribute(object): :type sparse_update: bool """ - def __init__(self, name=None, is_static=False, initial_std=None, - initial_mean=None, initial_max=None, initial_min=None, - l1_rate=None, l2_rate=None, learning_rate=None, momentum=None, + def __init__(self, + name=None, + is_static=False, + initial_std=None, + initial_mean=None, + initial_max=None, + initial_min=None, + l1_rate=None, + l2_rate=None, + learning_rate=None, + momentum=None, sparse_update=False): # initialize strategy. if is_static: @@ -183,7 +193,10 @@ class ExtraLayerAttribute(object): :type device: int """ - def __init__(self, error_clipping_threshold=None, drop_rate=None, device=None): + def __init__(self, + error_clipping_threshold=None, + drop_rate=None, + device=None): self.attr = dict() if isinstance(error_clipping_threshold, float): assert error_clipping_threshold > 0 @@ -200,8 +213,8 @@ class ExtraLayerAttribute(object): for key in self.attr: if not hasattr(self, 'can_%s' % key) or \ not getattr(self, 'can_%s' % key): - raise NotImplementedError( - "Layer %s cannot support %s" % (layer_name, key)) + raise NotImplementedError("Layer %s cannot support %s" % + (layer_name, key)) @staticmethod def to_kwargs(attr): diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index 283a45df30..b41097953d 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Data Sources are helpers to define paddle training data or testing data. """ @@ -26,8 +25,12 @@ except ImportError: __all__ = ['define_py_data_sources2'] -def define_py_data_source(file_list, cls, module, - obj, args=None, async=False, +def define_py_data_source(file_list, + cls, + module, + obj, + args=None, + async=False, data_cls=PyData): """ Define a python data source. @@ -76,8 +79,9 @@ def define_py_data_source(file_list, cls, module, args = pickle.dumps(args, 0) if data_cls is None: + def py_data2(files, load_data_module, load_data_object, load_data_args, - **kwargs): + **kwargs): data = DataBase() data.type = 'py2' data.files = files @@ -86,17 +90,25 @@ def define_py_data_source(file_list, cls, module, data.load_data_args = load_data_args data.async_load_data = True return data - data_cls = py_data2 - - cls(data_cls(files=file_list, - load_data_module=module, - load_data_object=obj, - load_data_args=args, - async_load_data=async)) + data_cls = py_data2 -def define_py_data_sources(train_list, test_list, module, obj, args=None, - train_async=False, data_cls=PyData): + cls( + data_cls( + files=file_list, + load_data_module=module, + load_data_object=obj, + load_data_args=args, + async_load_data=async)) + + +def define_py_data_sources(train_list, + test_list, + module, + obj, + args=None, + train_async=False, + data_cls=PyData): """ The annotation is almost the same as define_py_data_sources2, except that it can specific train_async and data_cls. @@ -125,8 +137,8 @@ def define_py_data_sources(train_list, test_list, module, obj, args=None, """ def __is_splitable__(o): - return (isinstance(o, list) or isinstance(o, tuple) - ) and hasattr(o, '__len__') and len(o) == 2 + return (isinstance(o, list) or + isinstance(o, tuple)) and hasattr(o, '__len__') and len(o) == 2 assert train_list is not None or test_list is not None assert module is not None and obj is not None @@ -196,9 +208,10 @@ def define_py_data_sources2(train_list, test_list, module, obj, args=None): :return: None :rtype: None """ - define_py_data_sources(train_list=train_list, - test_list=test_list, - module=module, - obj=obj, - args=args, - data_cls=None) + define_py_data_sources( + train_list=train_list, + test_list=test_list, + module=module, + obj=obj, + args=args, + data_cls=None) diff --git a/python/paddle/trainer_config_helpers/default_decorators.py b/python/paddle/trainer_config_helpers/default_decorators.py index be00f48b45..c01050e338 100644 --- a/python/paddle/trainer_config_helpers/default_decorators.py +++ b/python/paddle/trainer_config_helpers/default_decorators.py @@ -18,16 +18,18 @@ from .attrs import ParamAttr from .activations import TanhActivation from paddle.trainer.config_parser import * -__all__ = ['wrap_name_default', 'wrap_param_attr_default', - 'wrap_bias_attr_default', 'wrap_act_default', - 'wrap_param_default'] +__all__ = [ + 'wrap_name_default', 'wrap_param_attr_default', 'wrap_bias_attr_default', + 'wrap_act_default', 'wrap_param_default' +] def __default_not_set_callback__(kwargs, name): return name not in kwargs or kwargs[name] is None -def wrap_param_default(param_names=None, default_factory=None, +def wrap_param_default(param_names=None, + default_factory=None, not_set_callback=__default_not_set_callback__): assert param_names is not None assert isinstance(param_names, list) or isinstance(param_names, tuple) @@ -43,7 +45,8 @@ def wrap_param_default(param_names=None, default_factory=None, if argspec.defaults: num_positional -= len(argspec.defaults) if not argspec.varargs and len(args) > num_positional: - logger.fatal("Must use keyword arguments for non-positional args") + logger.fatal( + "Must use keyword arguments for non-positional args") for name in param_names: if not_set_callback(kwargs, name): # Not set kwargs[name] = default_factory(func) @@ -112,13 +115,13 @@ def wrap_param_attr_default(param_names=None, default_factory=None): return wrap_param_default(param_names, default_factory) -def wrap_bias_attr_default(param_names=None, default_factory=None, +def wrap_bias_attr_default(param_names=None, + default_factory=None, has_bias=True): if param_names is None: param_names = ['bias_attr'] if default_factory is None: - default_factory = lambda _: ParamAttr(initial_std=0., - initial_mean=0.) + default_factory = lambda _: ParamAttr(initial_std=0., initial_mean=0.) def __bias_attr_not_set__(kwargs, name): if has_bias: diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index ded124a5c8..dc6a36392f 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -15,13 +15,14 @@ from paddle.trainer.config_parser import * from default_decorators import * -__all__ = ["evaluator_base","classification_error_evaluator", "auc_evaluator", - "pnpair_evaluator", "precision_recall_evaluator", - "ctc_error_evaluator", "chunk_evaluator", "sum_evaluator", - "column_sum_evaluator", "value_printer_evaluator", - "gradient_printer_evaluator", "maxid_printer_evaluator", - "maxframe_printer_evaluator", "seqtext_printer_evaluator", - "classification_error_printer_evaluator"] +__all__ = [ + "evaluator_base", "classification_error_evaluator", "auc_evaluator", + "pnpair_evaluator", "precision_recall_evaluator", "ctc_error_evaluator", + "chunk_evaluator", "sum_evaluator", "column_sum_evaluator", + "value_printer_evaluator", "gradient_printer_evaluator", + "maxid_printer_evaluator", "maxframe_printer_evaluator", + "seqtext_printer_evaluator", "classification_error_printer_evaluator" +] class EvaluatorAttribute(object): @@ -32,10 +33,7 @@ class EvaluatorAttribute(object): FOR_UTILS = 1 << 4 KEYS = [ - "for_classification", - "for_regression", - "for_rank", - "for_print", + "for_classification", "for_regression", "for_rank", "for_print", "for_utils" ] @@ -55,22 +53,23 @@ def evaluator(*attrs): setattr(method, EvaluatorAttribute.to_key(attr), True) method.is_evaluator = True return method + return impl -def evaluator_base( - input, - type, - label=None, - weight=None, - name=None, - chunk_scheme=None, - num_chunk_types=None, - classification_threshold=None, - positive_label=None, - dict_file=None, - result_file=None, - num_results=None, - delimited=None): + +def evaluator_base(input, + type, + label=None, + weight=None, + name=None, + chunk_scheme=None, + num_chunk_types=None, + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None): """ Evaluator will evaluate the network status while training/testing. @@ -130,14 +129,14 @@ def evaluator_base( result_file=result_file, delimited=delimited) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() -def classification_error_evaluator( - input, - label, - name=None, - weight=None, - threshold=None): +def classification_error_evaluator(input, + label, + name=None, + weight=None, + threshold=None): """ Classification Error Evaluator. It will print error rate for classification. @@ -170,13 +169,14 @@ def classification_error_evaluator( :return: None. """ - evaluator_base(name=name, - type="classification_error", - input=input, - label=label, - weight=weight, - classification_threshold=threshold, - ) + evaluator_base( + name=name, + type="classification_error", + input=input, + label=label, + weight=weight, + classification_threshold=threshold, ) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -184,8 +184,7 @@ def auc_evaluator( input, label, name=None, - weight=None, - ): + weight=None, ): """ Auc Evaluator which adapts to binary classification. @@ -205,11 +204,13 @@ def auc_evaluator( [sample_num, 1]. :type weight: LayerOutput """ - evaluator_base(name=name, - type="last-column-auc", - input=input, - label=label, - weight=weight) + evaluator_base( + name=name, + type="last-column-auc", + input=input, + label=label, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_RANK) @wrap_name_default() @@ -218,8 +219,7 @@ def pnpair_evaluator( label, info, name=None, - weight=None, - ): + weight=None, ): """ Positive-negative pair rate Evaluator which adapts to rank task like learning to rank. This evaluator must contain at least three layers. @@ -242,12 +242,14 @@ def pnpair_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="pnpair", - input=input, - label=label, - info=info, - weight=weight) + evaluator_base( + name=name, + type="pnpair", + input=input, + label=label, + info=info, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -256,8 +258,7 @@ def precision_recall_evaluator( label, positive_label=None, weight=None, - name=None, - ): + name=None, ): """ An Evaluator to calculate precision and recall, F1-score. It is adapt to the task with multiple labels. @@ -286,20 +287,21 @@ def precision_recall_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="precision_recall", - input=input, - label=label, - positive_label=positive_label, - weight=weight) + evaluator_base( + name=name, + type="precision_recall", + input=input, + label=label, + positive_label=positive_label, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() def ctc_error_evaluator( input, label, - name=None, - ): + name=None, ): """ This evaluator is to calculate sequence-to-sequence edit distance. @@ -317,10 +319,9 @@ def ctc_error_evaluator( label for ctc_layer :type label: LayerOutput """ - evaluator_base(name=name, - type="ctc_edit_distance", - input=input, - label=label) + evaluator_base( + name=name, type="ctc_edit_distance", input=input, label=label) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -328,8 +329,7 @@ def chunk_evaluator( input, name=None, chunk_scheme=None, - num_chunk_types=None, - ): + num_chunk_types=None, ): """ Chunk evaluator is used to evaluate segment labelling accuracy for a sequence. It calculates the chunk detection F1 score. @@ -375,19 +375,20 @@ def chunk_evaluator( :type chunk_scheme: basestring :param num_chunk_types: number of chunk types other than "other" """ - evaluator_base(name=name, - type="chunk", - input=input, - chunk_scheme=chunk_scheme, - num_chunk_types=num_chunk_types) + evaluator_base( + name=name, + type="chunk", + input=input, + chunk_scheme=chunk_scheme, + num_chunk_types=num_chunk_types) + @evaluator(EvaluatorAttribute.FOR_UTILS) @wrap_name_default() def sum_evaluator( input, name=None, - weight=None, - ): + weight=None, ): """ An Evaluator to sum the result of input. @@ -405,18 +406,15 @@ def sum_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="sum", - input=input, - weight=weight) + evaluator_base(name=name, type="sum", input=input, weight=weight) + @evaluator(EvaluatorAttribute.FOR_UTILS) @wrap_name_default() def column_sum_evaluator( input, name=None, - weight=None, - ): + weight=None, ): """ This Evaluator is used to sum the last column of input. @@ -431,22 +429,22 @@ def column_sum_evaluator( :param input: Input Layer name. :type input: LayerOutput """ - evaluator_base(name=name, - type="last-column-sum", - input=input, - weight=weight) + evaluator_base( + name=name, type="last-column-sum", input=input, weight=weight) + """ The following are printer Evaluators which are usually used to print the result, like value or gradient of input layers, the results generated in machine translation, the classification error etc. """ + + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def value_printer_evaluator( input, - name=None, - ): + name=None, ): """ This Evaluator is used to print the values of input layers. It contains one or more input layers. @@ -462,16 +460,14 @@ def value_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="value_printer", - input=input) + evaluator_base(name=name, type="value_printer", input=input) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def gradient_printer_evaluator( input, - name=None, - ): + name=None, ): """ This Evaluator is used to print the gradient of input layers. It contains one or more input layers. @@ -487,17 +483,15 @@ def gradient_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="gradient_printer", - input=input) + evaluator_base(name=name, type="gradient_printer", input=input) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxid_printer_evaluator( input, num_results=None, - name=None, - ): + name=None, ): """ This Evaluator is used to print maximum top k values and their indexes of each row of input layers. It contains one or more input layers. @@ -517,18 +511,16 @@ def maxid_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="max_id_printer", - input=input, - num_results=num_results) + evaluator_base( + name=name, type="max_id_printer", input=input, num_results=num_results) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxframe_printer_evaluator( input, num_results=None, - name=None, - ): + name=None, ): """ This Evaluator is used to print the top k frames of each input layers. The input layers should contain sequences info or sequences type. @@ -549,10 +541,12 @@ def maxframe_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="max_frame_printer", - input=input, - num_results=num_results) + evaluator_base( + name=name, + type="max_frame_printer", + input=input, + num_results=num_results) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() @@ -562,8 +556,7 @@ def seqtext_printer_evaluator( id_input=None, dict_file=None, delimited=None, - name=None, - ): + name=None, ): """ Sequence text printer will print text according to index matrix and a dictionary. There can be multiple input to this layer: @@ -636,12 +629,14 @@ def seqtext_printer_evaluator( inputs = [id_input, input] input.parents.append(id_input) - evaluator_base(name=name, - type="seq_text_printer", - input=inputs, - dict_file=dict_file, - result_file=result_file, - delimited=delimited) + evaluator_base( + name=name, + type="seq_text_printer", + input=inputs, + dict_file=dict_file, + result_file=result_file, + delimited=delimited) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() @@ -649,8 +644,7 @@ def classification_error_printer_evaluator( input, label, threshold=0.5, - name=None, - ): + name=None, ): """ This Evaluator is used to print the classification error of each sample. @@ -667,8 +661,9 @@ def classification_error_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="classification_error_printer", - input=input, - label=label, - classification_threshold=threshold) + evaluator_base( + name=name, + type="classification_error_printer", + input=input, + label=label, + classification_threshold=threshold) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index a0a367f2d5..796121a641 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -29,36 +29,83 @@ except ImportError: import pickle import copy -__all__ = ["full_matrix_projection", "AggregateLevel", "ExpandLevel", - "identity_projection", "dotmul_projection", "dotmul_operator", - "repeat_layer", - "table_projection", "mixed_layer", "data_layer", - "embedding_layer", "fc_layer", "grumemory", - "pooling_layer", "lstmemory", "last_seq", "first_seq", - "cos_sim", "hsigmoid", "conv_projection", - "regression_cost", 'classification_cost', "LayerOutput", - 'img_conv_layer', 'img_pool_layer', 'batch_norm_layer', - 'img_cmrnorm_layer', 'addto_layer', - 'concat_layer', 'lstm_step_layer', 'recurrent_group', - 'memory', 'StaticInput', 'expand_layer', 'scaling_layer', - 'power_layer', 'interpolation_layer', 'bilinear_interp_layer', - 'trans_layer', 'sum_to_one_norm_layer', - 'get_output_layer', 'LayerType', 'context_projection', - 'beam_search', 'maxid_layer', 'GeneratedInput', 'SubsequenceInput', - 'gru_step_layer', 'recurrent_layer', - 'BaseGeneratedInput', 'conv_operator', 'conv_shift_layer', - 'tensor_layer', 'selective_fc_layer', 'sampling_id_layer', - 'slope_intercept_layer', 'trans_full_matrix_projection', - 'linear_comb_layer', - 'convex_comb_layer', 'ctc_layer', 'crf_layer', 'crf_decoding_layer', - 'nce_layer', - 'cross_entropy_with_selfnorm', 'cross_entropy', - 'multi_binary_label_cross_entropy', 'sum_cost', - 'rank_cost', 'lambda_cost', 'huber_cost', - 'block_expand_layer', - 'maxout_layer', 'out_prod_layer', 'print_layer', - 'spp_layer', - ] +__all__ = [ + "full_matrix_projection", + "AggregateLevel", + "ExpandLevel", + "identity_projection", + "dotmul_projection", + "dotmul_operator", + "repeat_layer", + "table_projection", + "mixed_layer", + "data_layer", + "embedding_layer", + "fc_layer", + "grumemory", + "pooling_layer", + "lstmemory", + "last_seq", + "first_seq", + "cos_sim", + "hsigmoid", + "conv_projection", + "regression_cost", + 'classification_cost', + "LayerOutput", + 'img_conv_layer', + 'img_pool_layer', + 'batch_norm_layer', + 'img_cmrnorm_layer', + 'addto_layer', + 'concat_layer', + 'lstm_step_layer', + 'recurrent_group', + 'memory', + 'StaticInput', + 'expand_layer', + 'scaling_layer', + 'power_layer', + 'interpolation_layer', + 'bilinear_interp_layer', + 'trans_layer', + 'sum_to_one_norm_layer', + 'get_output_layer', + 'LayerType', + 'context_projection', + 'beam_search', + 'maxid_layer', + 'GeneratedInput', + 'SubsequenceInput', + 'gru_step_layer', + 'recurrent_layer', + 'BaseGeneratedInput', + 'conv_operator', + 'conv_shift_layer', + 'tensor_layer', + 'selective_fc_layer', + 'sampling_id_layer', + 'slope_intercept_layer', + 'trans_full_matrix_projection', + 'linear_comb_layer', + 'convex_comb_layer', + 'ctc_layer', + 'crf_layer', + 'crf_decoding_layer', + 'nce_layer', + 'cross_entropy_with_selfnorm', + 'cross_entropy', + 'multi_binary_label_cross_entropy', + 'sum_cost', + 'rank_cost', + 'lambda_cost', + 'huber_cost', + 'block_expand_layer', + 'maxout_layer', + 'out_prod_layer', + 'print_layer', + 'spp_layer', +] class LayerType(object): @@ -181,8 +228,15 @@ class LayerOutput(object): :type parents: list|tuple|collections.Sequence """ - def __init__(self, name, layer_type, parents=None, activation=None, - num_filters=None, img_norm_type=None, size=None, outputs=None, + def __init__(self, + name, + layer_type, + parents=None, + activation=None, + num_filters=None, + img_norm_type=None, + size=None, + outputs=None, reverse=None): assert isinstance(name, basestring) assert isinstance(layer_type, basestring) @@ -223,6 +277,7 @@ DEVICE = 'device' def layer_support(*attrs): attrs_list = list(attrs) attrs_list.append(DEVICE) + def decorator(method): @functools.wraps(method) def wrapper(*args, **kwargs): @@ -282,9 +337,8 @@ def full_matrix_projection(input, size=0, param_attr=None): :return: A FullMatrixProjection Object. :rtype: FullMatrixProjection """ - proj = FullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = FullMatrixProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -319,9 +373,8 @@ def trans_full_matrix_projection(input, size=0, param_attr=None): :return: A TransposedFullMatrixProjection Object. :rtype: TransposedFullMatrixProjection """ - proj = TransposedFullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = TransposedFullMatrixProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -365,9 +418,8 @@ def table_projection(input, size=0, param_attr=None): :return: A TableProjection Object. :rtype: TableProjection """ - proj = TableProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = TableProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -413,8 +465,8 @@ def identity_projection(input, offset=None): proj = IdentityProjection(input_layer_name=input.name) proj.origin = input else: - proj = IdentityOffsetProjection(input_layer_name=input.name, - offset=offset) + proj = IdentityOffsetProjection( + input_layer_name=input.name, offset=offset) proj.origin = input return proj @@ -443,9 +495,8 @@ def dotmul_projection(input, param_attr=None): :return: A DotMulProjection Object. :rtype: DotMulProjection """ - proj = DotMulProjection(input_layer_name=input.name, - size=input.size, - **param_attr.attr) + proj = DotMulProjection( + input_layer_name=input.name, size=input.size, **param_attr.attr) proj.origin = input return proj @@ -478,21 +529,22 @@ def dotmul_operator(a=None, b=None, scale=1, **kwargs): if 'x' in kwargs or 'y' in kwargs: logger.warning('x and y arguments for dotmul_operator is deprecated. ' 'Please use a and b as parameter.') - a = kwargs.get('x', a) # For Backward capacity. + a = kwargs.get('x', a) # For Backward capacity. b = kwargs.get('y', b) assert isinstance(a, LayerOutput) assert isinstance(b, LayerOutput) if a.size is not None and b.size is not None: assert a.size == b.size - op = DotMulOperator(input_layer_names=[a.name, b.name], - scale=scale) + op = DotMulOperator(input_layer_names=[a.name, b.name], scale=scale) op.origin = [a, b] return op @wrap_bias_attr_default(['padding_attr']) -def context_projection(input, context_len, context_start=None, +def context_projection(input, + context_len, + context_start=None, padding_attr=False): """ Context Projection. @@ -529,11 +581,12 @@ def context_projection(input, context_len, context_start=None, if trainable: extra_dict = padding_attr.attr - proj = ContextProjection(input_layer_name=input.name, - context_length=context_len, - context_start=context_start, - trainable_padding=trainable, - **extra_dict) + proj = ContextProjection( + input_layer_name=input.name, + context_length=context_len, + context_start=context_start, + trainable_padding=trainable, + **extra_dict) proj.origin = input return proj @@ -547,8 +600,7 @@ class MixedLayerType(LayerOutput): def __init__(self): Exception.__init__(self) - def __init__(self, name, size, act, bias_attr, layer_attr, - parents=None): + def __init__(self, name, size, act, bias_attr, layer_attr, parents=None): """ Ctor. :param name: layer name. @@ -565,8 +617,13 @@ class MixedLayerType(LayerOutput): :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute or None """ - LayerOutput.__init__(self, name, LayerType.MIXED_LAYER, parents, - size=size, activation=act) + LayerOutput.__init__( + self, + name, + LayerType.MIXED_LAYER, + parents, + size=size, + activation=act) self.bias_attr = bias_attr self.layer_attr = layer_attr self.inputs = [] @@ -604,8 +661,7 @@ class MixedLayerType(LayerOutput): active_type=self.activation.name, bias=ParamAttr.to_bias(self.bias_attr), inputs=self.inputs, - **ExtraLayerAttribute.to_kwargs(self.layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(self.layer_attr)) # update the size which might be computed inside MixedLayer # according to the operator's output size self.size = ml.config.size @@ -615,7 +671,11 @@ class MixedLayerType(LayerOutput): @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(ERROR_CLIPPING, DROPOUT) -def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, +def mixed_layer(size=0, + input=None, + name=None, + act=None, + bias_attr=False, layer_attr=None): """ Mixed Layer. A mixed layer will add all inputs together, then activate. @@ -660,8 +720,12 @@ def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, if input is None: return MixedLayerType(name, size, act, bias_attr, layer_attr) else: - with mixed_layer(name=name, size=size, act=act, bias_attr=bias_attr, - layer_attr=layer_attr) as m: + with mixed_layer( + name=name, + size=size, + act=act, + bias_attr=bias_attr, + layer_attr=layer_attr) as m: if isinstance(input, collections.Sequence): for each in input: m += each @@ -691,8 +755,11 @@ def data_layer(name, size, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer(type=LayerType.DATA, name=name, size=size, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + type=LayerType.DATA, + name=name, + size=size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.DATA, size=size) @@ -718,9 +785,12 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - with mixed_layer(name=name, size=size, act=LinearActivation(), - bias_attr=False, - layer_attr=layer_attr) as mix: + with mixed_layer( + name=name, + size=size, + act=LinearActivation(), + bias_attr=False, + layer_attr=layer_attr) as mix: mix += table_projection(input=input, size=size, param_attr=param_attr) return mix @@ -730,8 +800,13 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): @wrap_bias_attr_default() @wrap_act_default() @layer_support(ERROR_CLIPPING, DROPOUT) -def fc_layer(input, size, act=None, name=None, - param_attr=None, bias_attr=None, layer_attr=None): +def fc_layer(input, + size, + act=None, + name=None, + param_attr=None, + bias_attr=None, + layer_attr=None): """ Helper for declare fully connected layer. @@ -783,17 +858,17 @@ def fc_layer(input, size, act=None, name=None, assert isinstance(input, collections.Sequence) Layer( - inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( - input, param_attr)], + inputs=[ + Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr) + ], name=name, type=LayerType.FC_LAYER, size=size, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.FC_LAYER, input, activation=act, - size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.FC_LAYER, input, activation=act, size=size) @wrap_name_default("print") @@ -816,8 +891,7 @@ def print_layer(input, name=None): Layer( name=name, type=LayerType.PRINT_LAYER, - inputs=[l.name for l in input], - ) + inputs=[l.name for l in input], ) # this layer don't return anything, can not be input of other layer. @@ -825,7 +899,10 @@ def print_layer(input, name=None): @wrap_bias_attr_default(has_bias=False) @wrap_param_default(['pooling_type'], default_factory=lambda _: MaxPooling()) @layer_support() -def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, +def pooling_layer(input, + pooling_type=None, + name=None, + bias_attr=None, agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ @@ -872,24 +949,27 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, inputs=[Input(input.name)], bias=ParamAttr.to_bias(bias_attr), trans_type=agg_level, - **extra_dict - ) - - return LayerOutput(name, pooling_type.name, parents=[input], - size=input.size) + **extra_dict) + return LayerOutput( + name, pooling_type.name, parents=[input], size=input.size) @wrap_bias_attr_default() @wrap_param_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(param_names=["act", 'state_act'], act=TanhActivation()) @wrap_name_default("lstmemory") @layer_support(DROPOUT) -def lstmemory(input, name=None, reverse=False, act=None, - gate_act=None, size=None, - state_act=None, bias_attr=None, param_attr=None, +def lstmemory(input, + name=None, + reverse=False, + act=None, + gate_act=None, + size=None, + state_act=None, + bias_attr=None, + param_attr=None, layer_attr=None): """ Long Short-term Memory Cell. @@ -964,30 +1044,38 @@ def lstmemory(input, name=None, reverse=False, act=None, "layer. The lstm size should be equal with input layer size/4. The" " size which is set explicitly will be ignored." % name) - Layer(name=name, - type=LayerType.LSTMEMORY, - active_type=act.name, - active_state_type=state_act.name, - active_gate_type=gate_act.name, - reversed=reverse, - bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input.name, **param_attr.attr)], - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type=LayerType.LSTMEMORY, + active_type=act.name, + active_state_type=state_act.name, + active_gate_type=gate_act.name, + reversed=reverse, + bias=ParamAttr.to_bias(bias_attr), + inputs=[Input(input.name, **param_attr.attr)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.LSTMEMORY, [input], size=input.size / 4, - reverse=reverse) + return LayerOutput( + name, + LayerType.LSTMEMORY, [input], + size=input.size / 4, + reverse=reverse) @wrap_bias_attr_default() @wrap_param_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(param_names=["act"], act=TanhActivation()) @wrap_name_default("gru") @layer_support(DROPOUT) -def grumemory(input, name=None, reverse=False, act=None, - gate_act=None, size=None, - bias_attr=None, param_attr=None, +def grumemory(input, + name=None, + reverse=False, + act=None, + gate_act=None, + size=None, + bias_attr=None, + param_attr=None, layer_attr=None): """ Gate Recurrent Unit Layer. @@ -1078,23 +1166,28 @@ def grumemory(input, name=None, reverse=False, act=None, " and should be input size / 3. Set size explicitly will be " "ignored.") - Layer(name=name, - type=LayerType.GRUMEMORY, - active_type=act.name, - active_gate_type=gate_act.name, - reversed=reverse, - bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input.name, **param_attr.attr)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + Layer( + name=name, + type=LayerType.GRUMEMORY, + active_type=act.name, + active_gate_type=gate_act.name, + reversed=reverse, + bias=ParamAttr.to_bias(bias_attr), + inputs=[Input(input.name, **param_attr.attr)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.GRUMEMORY, [input], size=input.size / 3, - reverse=reverse) + return LayerOutput( + name, + LayerType.GRUMEMORY, [input], + size=input.size / 3, + reverse=reverse) @wrap_name_default() @layer_support() -def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, +def last_seq(input, + name=None, + agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ Get Last Timestamp Activation of a sequence. @@ -1120,15 +1213,19 @@ def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, type=LayerType.SEQUENCE_LAST_INSTANCE, inputs=[input.name], trans_type=agg_level, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEQUENCE_LAST_INSTANCE, parents=[input], - size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEQUENCE_LAST_INSTANCE, + parents=[input], + size=input.size) @wrap_name_default() @layer_support() -def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, +def first_seq(input, + name=None, + agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ Get First Timestamp Activation of a sequence. @@ -1155,10 +1252,12 @@ def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, type=LayerType.SEQUENCE_FIRST_INSTANCE, inputs=[input.name], trans_type=agg_level, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEQUENCE_FIRST_INSTANCE, - parents=[input], size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEQUENCE_FIRST_INSTANCE, + parents=[input], + size=input.size) class ExpandLevel(object): @@ -1168,7 +1267,8 @@ class ExpandLevel(object): @wrap_name_default() @layer_support() -def expand_layer(input, expand_as, +def expand_layer(input, + expand_as, name=None, bias_attr=False, expand_level=ExpandLevel.FROM_TIMESTEP, @@ -1208,19 +1308,17 @@ def expand_layer(input, expand_as, bias=ParamAttr.to_bias(bias_attr=bias_attr), type=LayerType.EXPAND_LAYER, trans_type=expand_level, - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name=name, - size=input.size, - layer_type=LayerType.EXPAND_LAYER, - parents=[input, expand_as]) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + size=input.size, + layer_type=LayerType.EXPAND_LAYER, + parents=[input, expand_as]) @wrap_name_default() @layer_support() -def repeat_layer(input, num_repeats, - name=None, - layer_attr=None): +def repeat_layer(input, num_repeats, name=None, layer_attr=None): """ A layer for repeating the input for num_repeats times. This is equivalent to apply concat_layer() with num_repeats same input. @@ -1251,12 +1349,13 @@ def repeat_layer(input, num_repeats, name=name, num_filters=num_repeats, type=LayerType.FEATURE_MAP_EXPAND_LAYER, - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name=name, - size=l.config.size, - layer_type=LayerType.FEATURE_MAP_EXPAND_LAYER, - parents=[input]) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + size=l.config.size, + layer_type=LayerType.FEATURE_MAP_EXPAND_LAYER, + parents=[input]) + @wrap_name_default() @layer_support() @@ -1302,11 +1401,12 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.INTERPOLATION_LAYER, inputs=[weight.name, input[0].name, input[1].name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.INTERPOLATION_LAYER, - parents=[weight, input[0], input[1]], - size=input[0].size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.INTERPOLATION_LAYER, + parents=[weight, input[0], input[1]], + size=input[0].size) @wrap_name_default() @@ -1345,15 +1445,23 @@ def bilinear_interp_layer(input, assert out_size_x > 0 and out_size_y > 0 assert input.num_filters is not None num_channels = input.num_filters - l = Layer(name=name, - inputs=Input(input.name, - bilinear_interp=BilinearInterp(out_size_x=out_size_x, - out_size_y=out_size_y, - num_channels=num_channels)), - type=LayerType.BILINEAR_INTERP_LAYER, - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input], - num_filters=num_channels, size=l.config.size) + l = Layer( + name=name, + inputs=Input( + input.name, + bilinear_interp=BilinearInterp( + out_size_x=out_size_x, + out_size_y=out_size_y, + num_channels=num_channels)), + type=LayerType.BILINEAR_INTERP_LAYER, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.BILINEAR_INTERP_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + @wrap_name_default() @layer_support() @@ -1392,10 +1500,9 @@ def power_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.POWER_LAYER, inputs=[weight.name, input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.POWER_LAYER, - parents=[input, weight], size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.POWER_LAYER, parents=[input, weight], size=input.size) @wrap_name_default() @@ -1437,10 +1544,9 @@ def scaling_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.SCALING_LAYER, inputs=[weight.name, input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SCALING_LAYER, parents=[weight, input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SCALING_LAYER, parents=[weight, input], size=input.size) @wrap_name_default() @@ -1473,10 +1579,9 @@ def trans_layer(input, name=None, layer_attr=None): name=name, type=LayerType.TRANS_LAYER, inputs=[input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.TRANS_LAYER, parents=[input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.TRANS_LAYER, parents=[input], size=input.size) @wrap_name_default() @@ -1518,8 +1623,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): type=LayerType.COSINE_SIM, cos_scale=scale, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) else: if a.size is not None and b.size is not None: assert size == b.size / a.size @@ -1529,8 +1633,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): size=size, cos_scale=scale, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b], size=size) @@ -1538,8 +1641,13 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): @wrap_bias_attr_default(has_bias=True) @wrap_param_attr_default() @layer_support() -def hsigmoid(input, label, num_classes, name=None, bias_attr=None, - param_attr=None, layer_attr=None): +def hsigmoid(input, + label, + num_classes, + name=None, + bias_attr=None, + param_attr=None, + layer_attr=None): """ Organize the classes into a binary tree. At each node, a sigmoid function is used to calculate the probability of belonging to the right branch. @@ -1600,10 +1708,9 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, num_classes=num_classes, bias=ParamAttr.to_bias(bias_attr), inputs=ipts_for_layer, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.HSIGMOID, parents=parents, - size=l.config.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.HSIGMOID, parents=parents, size=l.config.size) @wrap_name_default("conv") @@ -1611,11 +1718,22 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, @wrap_bias_attr_default() @wrap_act_default(act=ReluActivation()) @layer_support(DROPOUT) -def img_conv_layer(input, filter_size, num_filters, - name=None, num_channels=None, - act=None, groups=1, stride=1, padding=0, bias_attr=None, - param_attr=None, shared_biases=True, layer_attr=None, - filter_size_y=None, stride_y=None, padding_y=None, +def img_conv_layer(input, + filter_size, + num_filters, + name=None, + num_channels=None, + act=None, + groups=1, + stride=1, + padding=0, + bias_attr=None, + param_attr=None, + shared_biases=True, + layer_attr=None, + filter_size_y=None, + stride_y=None, + padding_y=None, trans=False): """ Convolution layer for image. Paddle only support square input currently and @@ -1713,40 +1831,56 @@ def img_conv_layer(input, filter_size, num_filters, if param_attr.attr.get('initial_smart'): # special initial for conv layers. - init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + init_w = (2.0 / (filter_size**2 * num_channels))**0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False - + lt = LayerType.CONVTRANS_LAYER if trans else LayerType.CONV_LAYER - + l = Layer( name=name, - inputs=Input(input.name, conv=Conv( - filter_size=filter_size, padding=padding, stride=stride, - channels=num_channels, groups=groups, - filter_size_y=filter_size_y, padding_y=padding_y, - stride_y=stride_y), - **param_attr.attr), + inputs=Input( + input.name, + conv=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + groups=groups, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y), + **param_attr.attr), active_type=act.name, num_filters=num_filters, bias=ParamAttr.to_bias(bias_attr), shared_biases=shared_biases, type=lt, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, lt, parents=[input], - activation=act, num_filters=num_filters, - size=l.config.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + lt, + parents=[input], + activation=act, + num_filters=num_filters, + size=l.config.size) @wrap_name_default("pool") @layer_support() -def img_pool_layer(input, pool_size, name=None, - num_channels=None, pool_type=None, - stride=1, padding=0, layer_attr=None, - pool_size_y=None, stride_y=None, padding_y=None, +def img_pool_layer(input, + pool_size, + name=None, + num_channels=None, + pool_type=None, + stride=1, + padding=0, + layer_attr=None, + pool_size_y=None, + stride_y=None, + padding_y=None, img_width=None): """ Image pooling Layer. @@ -1804,29 +1938,39 @@ def img_pool_layer(input, pool_size, name=None, l = Layer( name=name, type=LayerType.POOL_LAYER, - inputs=[Input(input.name, - pool=Pool( - pool_type=type_name, - channels=num_channels, - size_x=pool_size, - start=None, - stride=stride, - padding=padding, - size_y=pool_size_y, - stride_y=stride_y, - padding_y=padding_y, - img_width=img_width - ))], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.POOL_LAYER, parents=[input], - num_filters=num_channels, size=l.config.size) + inputs=[ + Input( + input.name, + pool=Pool( + pool_type=type_name, + channels=num_channels, + size_x=pool_size, + start=None, + stride=stride, + padding=padding, + size_y=pool_size_y, + stride_y=stride_y, + padding_y=padding_y, + img_width=img_width)) + ], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.POOL_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) @wrap_name_default("spp") @layer_support() -def spp_layer(input, name=None, num_channels=None, pool_type=None, - pyramid_height=None, img_width=None, layer_attr=None): +def spp_layer(input, + name=None, + num_channels=None, + pool_type=None, + pyramid_height=None, + img_width=None, + layer_attr=None): """ Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition. The details please refer to @@ -1866,42 +2010,58 @@ def spp_layer(input, name=None, num_channels=None, pool_type=None, l = Layer( name=name, type=LayerType.SPP_LAYER, - inputs=Input(input.name, - spp=SpatialPyramidPool(pool_type=type_name, - channels=num_channels, - pyramid_height=pyramid_height, - img_width=img_width) - ), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, layer_type=LayerType.SPP_LAYER, parents=[input], - num_filters=num_channels, size=l.config.size) - - -def __img_norm_layer__(name, input, size, norm_type, scale, power, - num_channels, blocked, layer_attr): + inputs=Input( + input.name, + spp=SpatialPyramidPool( + pool_type=type_name, + channels=num_channels, + pyramid_height=pyramid_height, + img_width=img_width)), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.SPP_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + + +def __img_norm_layer__(name, input, size, norm_type, scale, power, num_channels, + blocked, layer_attr): if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters l = Layer( - name=name, type=LayerType.NORM_LAYER, inputs=Input( - input.name, norm=Norm(norm_type=norm_type, - channels=num_channels, size=size, - scale=scale, - pow=power, blocked=blocked) - ), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, layer_type=LayerType.NORM_LAYER, parents=[input], - num_filters=num_channels, img_norm_type=norm_type, - size=l.config.size) + name=name, + type=LayerType.NORM_LAYER, + inputs=Input( + input.name, + norm=Norm( + norm_type=norm_type, + channels=num_channels, + size=size, + scale=scale, + pow=power, + blocked=blocked)), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.NORM_LAYER, + parents=[input], + num_filters=num_channels, + img_norm_type=norm_type, + size=l.config.size) @wrap_name_default("crmnorm") @layer_support() -def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, - name=None, num_channels=None, +def img_cmrnorm_layer(input, + size, + scale=0.0128, + power=0.75, + name=None, + num_channels=None, layer_attr=None): """ Response normalization across feature maps. @@ -1935,8 +2095,13 @@ def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, @wrap_act_default(act=ReluActivation()) @wrap_name_default("batch_norm") @layer_support(DROPOUT) -def batch_norm_layer(input, act=None, name=None, num_channels=None, - bias_attr=None, param_attr=None, layer_attr=None, +def batch_norm_layer(input, + act=None, + name=None, + num_channels=None, + bias_attr=None, + param_attr=None, + layer_attr=None, batch_norm_type=None, moving_average_fraction=0.9, use_global_stats=None): @@ -2022,22 +2187,23 @@ def batch_norm_layer(input, act=None, name=None, num_channels=None, (batch_norm_type == "cudnn_batch_norm") l = Layer( name=name, - inputs=Input(input.name, - image=Image(channels=num_channels), - **param_attr.attr), + inputs=Input( + input.name, image=Image(channels=num_channels), **param_attr.attr), active_type=act.name, type=LayerType.BATCH_NORM_LAYER, batch_norm_type=batch_norm_type, bias=ParamAttr.to_bias(bias_attr), moving_average_fraction=moving_average_fraction, use_global_stats=use_global_stats, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.BATCH_NORM_LAYER, - parents=[input], activation=act, - num_filters=num_channels, - size=l.config.size) + return LayerOutput( + name=name, + layer_type=LayerType.BATCH_NORM_LAYER, + parents=[input], + activation=act, + num_filters=num_channels, + size=l.config.size) @wrap_name_default() @@ -2072,18 +2238,16 @@ def sum_to_one_norm_layer(input, name=None, layer_attr=None): name=name, type=LayerType.SUM_TO_ONE_NORM_LAYER, inputs=[input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], size=input.size) @wrap_name_default("addto") @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(DROPOUT) -def addto_layer(input, act=None, name=None, bias_attr=None, - layer_attr=None): +def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): """ AddtoLayer. @@ -2143,15 +2307,20 @@ def addto_layer(input, act=None, name=None, bias_attr=None, num_filters = each_input.num_filters l = Layer( - name=name, type=LayerType.ADDTO_LAYER, inputs=ipts_for_layer, + name=name, + type=LayerType.ADDTO_LAYER, + inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.ADDTO_LAYER, parents=input, - activation=act, num_filters=num_filters, - size=l.config.size) + return LayerOutput( + name, + LayerType.ADDTO_LAYER, + parents=input, + activation=act, + num_filters=num_filters, + size=l.config.size) @wrap_act_default(act=IdentityActivation()) @@ -2210,22 +2379,22 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): LayerOutput) return a - is_concat_layer = __is_type__(reduce(__reduce_concat_type__, - map(type, input)), LayerOutput) + is_concat_layer = __is_type__( + reduce(__reduce_concat_type__, map(type, input)), LayerOutput) - layer_type = (LayerType.CONCAT_LAYER if is_concat_layer - else LayerType.CONCAT_PROJ_LAYER) + layer_type = (LayerType.CONCAT_LAYER + if is_concat_layer else LayerType.CONCAT_PROJ_LAYER) if layer_type == LayerType.CONCAT_LAYER: assert not bias_attr Layer( - name=name, type=layer_type, + name=name, + type=layer_type, inputs=[x.name for x in input] if is_concat_layer else input, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) sz = 0 for each_input in input: @@ -2235,14 +2404,20 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): sz = None break - return LayerOutput(name, layer_type=layer_type, - parents=input if is_concat_layer else [ - x.origin for x in input], - activation=act, size=sz) - - -def memory(name, size, is_seq=False, boot_layer=None, - boot_bias=None, boot_bias_active_type=None, + return LayerOutput( + name, + layer_type=layer_type, + parents=input if is_concat_layer else [x.origin for x in input], + activation=act, + size=sz) + + +def memory(name, + size, + is_seq=False, + boot_layer=None, + boot_bias=None, + boot_bias_active_type=None, boot_with_const_id=None): """ The memory layers is a layer cross each time step. Reference this output @@ -2290,30 +2465,33 @@ def memory(name, size, is_seq=False, boot_layer=None, assert boot_layer is None or isinstance(boot_layer, LayerOutput) - agent_name = Memory(name, size, - is_seq, - boot_layer.name if boot_layer is not None else None, - boot_bias, - boot_bias_active_type.name, - boot_with_const_id) - - lout = LayerOutput(name=agent_name, size=size, - layer_type=LayerType.MEMORY, - parents=[boot_layer] if boot_layer is not None - else None) + agent_name = Memory(name, size, is_seq, boot_layer.name + if boot_layer is not None else None, boot_bias, + boot_bias_active_type.name, boot_with_const_id) + + lout = LayerOutput( + name=agent_name, + size=size, + layer_type=LayerType.MEMORY, + parents=[boot_layer] if boot_layer is not None else None) return lout @wrap_bias_attr_default() -@wrap_act_default(param_names=['gate_act', - 'state_act'], - act=SigmoidActivation()) +@wrap_act_default( + param_names=['gate_act', 'state_act'], act=SigmoidActivation()) @wrap_act_default(act=TanhActivation()) @wrap_name_default('lstm_step') @layer_support() -def lstm_step_layer(input, state, size, act=None, - name=None, gate_act=None, state_act=None, - bias_attr=None, layer_attr=None): +def lstm_step_layer(input, + state, + size, + act=None, + name=None, + gate_act=None, + state_act=None, + bias_attr=None, + layer_attr=None): """ LSTM Step Layer. It used in recurrent_group. The lstm equations are shown as follow. @@ -2380,24 +2558,32 @@ def lstm_step_layer(input, state, size, act=None, active_gate_type=gate_act.name, active_state_type=state_act.name, bias=ParamAttr.to_bias(bias_attr), - size=size, inputs=[input.name, state.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + size=size, + inputs=[input.name, state.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.LSTM_STEP_LAYER, - parents=[input, state], activation=act, - size=size, outputs=['default', 'state']) + return LayerOutput( + name=name, + layer_type=LayerType.LSTM_STEP_LAYER, + parents=[input, state], + activation=act, + size=size, + outputs=['default', 'state']) @wrap_bias_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(act=TanhActivation()) @wrap_name_default('gru_step') @layer_support() -def gru_step_layer(input, output_mem, size=None, act=None, - name=None, gate_act=None, - bias_attr=None, layer_attr=None): +def gru_step_layer(input, + output_mem, + size=None, + act=None, + name=None, + gate_act=None, + bias_attr=None, + layer_attr=None): """ :param input: @@ -2418,20 +2604,18 @@ def gru_step_layer(input, output_mem, size=None, act=None, Layer( name=name, type=LayerType.GRU_STEP_LAYER, - inputs=[ - input.name, - output_mem.name - ], + inputs=[input.name, output_mem.name], bias=ParamAttr.to_bias(bias_attr), size=size, active_type=act.name, active_gate_type=gate_act.name, - **ExtraAttr.to_kwargs(layer_attr) - ) + **ExtraAttr.to_kwargs(layer_attr)) return LayerOutput( - name=name, layer_type=LayerType.GRU_STEP_LAYER, + name=name, + layer_type=LayerType.GRU_STEP_LAYER, parents=[input, output_mem], - size=size, activation=act) + size=size, + activation=act) @wrap_name_default() @@ -2459,13 +2643,19 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): ' The get output name is %s, which not' \ ' in %s' % ( arg_name, ",".join(input.outputs)) - Layer(name=name, type=LayerType.GET_OUTPUT_LAYER, - inputs=[Input(input.name, input_layer_argument=arg_name)], - size=input.size, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type=LayerType.GET_OUTPUT_LAYER, + inputs=[Input( + input.name, input_layer_argument=arg_name)], + size=input.size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.GET_OUTPUT_LAYER, - parents=[input], size=input.size) + return LayerOutput( + name=name, + layer_type=LayerType.GET_OUTPUT_LAYER, + parents=[input], + size=input.size) @wrap_name_default() @@ -2473,8 +2663,13 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): @wrap_bias_attr_default() @wrap_param_attr_default() @layer_support() -def recurrent_layer(input, act=None, bias_attr=None, - param_attr=None, name=None, reverse=False, layer_attr=None): +def recurrent_layer(input, + act=None, + bias_attr=None, + param_attr=None, + name=None, + reverse=False, + layer_attr=None): """ Simple recurrent unit layer. It is just a fully connect layer through both time and neural network. @@ -2509,16 +2704,21 @@ def recurrent_layer(input, act=None, bias_attr=None, :return: LayerOutput object. :rtype: LayerOutput """ - Layer(name=name, - type=LayerType.RECURRENT_LAYER, - inputs=Input(input.name, **param_attr.attr), - active_type=act.name, - bias=ParamAttr.to_bias(bias_attr), - reversed=reverse, - **ExtraAttr.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.RECURRENT_LAYER, - parents=[input], size=input.size, activation=act, - reverse=reverse) + Layer( + name=name, + type=LayerType.RECURRENT_LAYER, + inputs=Input(input.name, **param_attr.attr), + active_type=act.name, + bias=ParamAttr.to_bias(bias_attr), + reversed=reverse, + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.RECURRENT_LAYER, + parents=[input], + size=input.size, + activation=act, + reverse=reverse) class StaticInput(object): @@ -2646,7 +2846,7 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): return True return False - assert(targetInlink == None or targetInlink_in_inlinks()) + assert (targetInlink == None or targetInlink_in_inlinks()) targetInlinkName = None if targetInlink == None \ else targetInlink.name if isinstance(targetInlink, LayerOutput) \ else targetInlink.input.name @@ -2661,7 +2861,8 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): return x.name RecurrentLayerGroupWithoutOutLinksBegin( - name=name, in_links=map(map_in_links, in_links), + name=name, + in_links=map(map_in_links, in_links), seq_reversed=reverse, target_inlinkname=targetInlinkName) in_args = [] @@ -2673,12 +2874,15 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): in_args.append(each_input.input) else: mem_name = "__%s_memory__" % each_input.input.name - mem = memory(name=mem_name, - is_seq=each_input.is_seq, - size=each_input.input.size, - boot_layer=each_input.input) - with mixed_layer(name=mem_name, size=each_input.input.size, - act=IdentityActivation()) as mix: + mem = memory( + name=mem_name, + is_seq=each_input.is_seq, + size=each_input.input.size, + boot_layer=each_input.input) + with mixed_layer( + name=mem_name, + size=each_input.input.size, + act=IdentityActivation()) as mix: mix += identity_projection(mem) in_args.append(mem) @@ -2720,14 +2924,15 @@ class GeneratedInput(BaseGeneratedInput): return maxid_layer(input=input, name='__beam_search_predict__') def before_real_step(self): - predict_id = memory(name='__beam_search_predict__', - size=self.size, - boot_with_const_id=self.bos_id) - - trg_emb = embedding_layer(input=predict_id, - size=self.embedding_size, - param_attr=ParamAttr( - name=self.embedding_name)) + predict_id = memory( + name='__beam_search_predict__', + size=self.size, + boot_with_const_id=self.bos_id) + + trg_emb = embedding_layer( + input=predict_id, + size=self.embedding_size, + param_attr=ParamAttr(name=self.embedding_name)) return trg_emb def __init__(self, size, embedding_name, embedding_size): @@ -2760,14 +2965,16 @@ def maxid_layer(input, name=None, layer_attr=None): """ assert isinstance(input, LayerOutput) - l = Layer(name=name, - type='maxid', - inputs=[input.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, - layer_type=LayerType.MAXID_LAYER, - parents=[input], - size=l.config.size) + l = Layer( + name=name, + type='maxid', + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.MAXID_LAYER, + parents=[input], + size=l.config.size) @wrap_name_default() @@ -2796,14 +3003,16 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): assert isinstance(input1, LayerOutput) assert isinstance(input2, LayerOutput) - l = Layer(name=name, - type=LayerType.OUT_PROD_LAYER, - inputs=[input1.name, input2.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, - layer_type=LayerType.OUT_PROD_LAYER, - parents=[input1, input2], - size=l.config.size) + l = Layer( + name=name, + type=LayerType.OUT_PROD_LAYER, + inputs=[input1.name, input2.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.OUT_PROD_LAYER, + parents=[input1, input2], + size=l.config.size) @wrap_name_default() @@ -2832,19 +3041,27 @@ def eos_layer(input, eos_id, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - l = Layer(name=name, - type=LayerType.EOSID_LAYER, - eos_id=eos_id, - inputs=[input.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.EOSID_LAYER, - parents=[input], - size=l.config.size) + l = Layer( + name=name, + type=LayerType.EOSID_LAYER, + eos_id=eos_id, + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.EOSID_LAYER, + parents=[input], + size=l.config.size) @wrap_name_default() -def beam_search(step, input, bos_id, eos_id, beam_size, - max_length=500, name=None, +def beam_search(step, + input, + bos_id, + eos_id, + beam_size, + max_length=500, + name=None, num_results_per_sample=None): """ Beam search is a heuristic search algorithm used in sequence generation. @@ -2918,8 +3135,7 @@ def beam_search(step, input, bos_id, eos_id, beam_size, if num_results_per_sample > beam_size: logger.warning("num_results_per_sample should be less than beam_size") - if isinstance(input, StaticInput) or isinstance(input, - BaseGeneratedInput): + if isinstance(input, StaticInput) or isinstance(input, BaseGeneratedInput): input = [input] generated_input_index = -1 @@ -2944,11 +3160,12 @@ def beam_search(step, input, bos_id, eos_id, beam_size, def __real_step__(*args): eos_name = "__%s_eos_layer__" % name - RecurrentLayerGroupSetGenerator(Generator( - eos_layer_name=eos_name, - max_num_frames=max_length, - beam_size=beam_size, - num_results_per_sample=num_results_per_sample)) + RecurrentLayerGroupSetGenerator( + Generator( + eos_layer_name=eos_name, + max_num_frames=max_length, + beam_size=beam_size, + num_results_per_sample=num_results_per_sample)) args = list(args) args.insert(generated_input_index, gipt.before_real_step()) @@ -2959,11 +3176,12 @@ def beam_search(step, input, bos_id, eos_id, beam_size, return predict - tmp = recurrent_group(step=__real_step__, input=real_input, reverse=False, - name=name) + tmp = recurrent_group( + step=__real_step__, input=real_input, reverse=False, name=name) return tmp + def __cost_input__(input, label, weight=None): """ inputs and parents for cost layers. @@ -2979,8 +3197,7 @@ def __cost_input__(input, label, weight=None): @wrap_name_default() @layer_support() -def regression_cost(input, label, weight=None, name=None, - layer_attr=None): +def regression_cost(input, label, weight=None, name=None, layer_attr=None): """ Regression Layer. @@ -3002,14 +3219,20 @@ def regression_cost(input, label, weight=None, name=None, """ ipts, parents = __cost_input__(input, label, weight) - Layer(inputs=ipts, type="square_error", name=name, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + inputs=ipts, + type="square_error", + name=name, + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.COST, parents=parents, size=1) @wrap_name_default("cost") @layer_support() -def classification_cost(input, label, weight=None, name=None, +def classification_cost(input, + label, + weight=None, + name=None, evaluator=classification_error_evaluator, layer_attr=None): """ @@ -3036,8 +3259,11 @@ def classification_cost(input, label, weight=None, name=None, ipts, parents = __cost_input__(input, label, weight) - Layer(name=name, type="multi-class-cross-entropy", inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type="multi-class-cross-entropy", + inputs=ipts, + **ExtraLayerAttribute.to_kwargs(layer_attr)) def __add_evaluator__(e): assert callable(e) @@ -3059,9 +3285,16 @@ def classification_cost(input, label, weight=None, name=None, return LayerOutput(name, LayerType.COST, parents=parents, size=1) -def conv_operator(img, filter, filter_size, num_filters, - num_channels=None, stride=1, padding=0, - filter_size_y=None, stride_y=None, padding_y=None): +def conv_operator(img, + filter, + filter_size, + num_filters, + num_channels=None, + stride=1, + padding=0, + filter_size_y=None, + stride_y=None, + padding_y=None): """ Different from img_conv_layer, conv_op is an Operator, which can be used in mixed_layer. And conv_op takes two inputs to perform convolution. @@ -3117,24 +3350,34 @@ def conv_operator(img, filter, filter_size, num_filters, if filter.size is not None: filter.size = filter_size * filter_size_y * num_filters * num_channels - op = ConvOperator(input_layer_names=[img.name, filter.name], - num_filters=num_filters, - conv_conf=Conv(filter_size=filter_size, - padding=padding, - stride=stride, - channels=num_channels, - filter_size_y=filter_size_y, - padding_y=padding_y, - stride_y=stride_y, - groups=1)) + op = ConvOperator( + input_layer_names=[img.name, filter.name], + num_filters=num_filters, + conv_conf=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=1)) op.origin = [img, filter] return op + @wrap_param_attr_default() -def conv_projection(input, filter_size, num_filters, - num_channels=None, stride=1, padding=0, - filter_size_y=None, stride_y=None, padding_y=None, - groups=1, param_attr=None): +def conv_projection(input, + filter_size, + num_filters, + num_channels=None, + stride=1, + padding=0, + filter_size_y=None, + stride_y=None, + padding_y=None, + groups=1, + param_attr=None): """ ConvProjection with a layer as input. It performs element-wise multiplication with weight. @@ -3206,23 +3449,25 @@ def conv_projection(input, filter_size, num_filters, if param_attr.attr.get('initial_smart'): # special initial for conv layers. - init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + init_w = (2.0 / (filter_size**2 * num_channels))**0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False - proj = ConvProjection(input_layer_name=input.name, - num_filters=num_filters, - conv_conf=Conv(filter_size=filter_size, - padding=padding, - stride=stride, - channels=num_channels, - filter_size_y=filter_size_y, - padding_y=padding_y, - stride_y=stride_y, - groups=groups), - **param_attr.attr) + proj = ConvProjection( + input_layer_name=input.name, + num_filters=num_filters, + conv_conf=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=groups), + **param_attr.attr) proj.origin = input return proj @@ -3270,11 +3515,10 @@ def conv_shift_layer(a, b, name=None, layer_attr=None): name=name, type=LayerType.CONV_SHIFT_LAYER, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], - size=a.size) + return LayerOutput( + name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], size=a.size) @wrap_name_default() @@ -3282,8 +3526,14 @@ def conv_shift_layer(a, b, name=None, layer_attr=None): @wrap_bias_attr_default() @wrap_act_default(act=LinearActivation()) @layer_support(ERROR_CLIPPING, DROPOUT) -def tensor_layer(a, b, size, act=None, name=None, - param_attr=None, bias_attr=None, layer_attr=None): +def tensor_layer(a, + b, + size, + act=None, + name=None, + param_attr=None, + bias_attr=None, + layer_attr=None): """ This layer performs tensor operation for two input. For example, each sample: @@ -3332,12 +3582,10 @@ def tensor_layer(a, b, size, act=None, name=None, type=LayerType.TENSOR_LAYER, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(a.name, **param_attr.attr), - Input(b.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.TENSOR_LAYER, parents=[a, b], - activation=act, size=size) + inputs=[Input(a.name, **param_attr.attr), Input(b.name)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.TENSOR_LAYER, parents=[a, b], activation=act, size=size) @wrap_name_default() @@ -3345,11 +3593,17 @@ def tensor_layer(a, b, size, act=None, name=None, @wrap_bias_attr_default() @wrap_act_default() @layer_support() -def selective_fc_layer(input, select, size, act=None, name=None, +def selective_fc_layer(input, + select, + size, + act=None, + name=None, pass_generation=False, has_selected_colums=True, mul_ratio=0.02, - param_attr=None, bias_attr=None, layer_attr=None): + param_attr=None, + bias_attr=None, + layer_attr=None): """ Selectived fully connected layer. Different from fc_layer, the output of this layer maybe sparse. It requires an additional input to indicate @@ -3399,8 +3653,9 @@ def selective_fc_layer(input, select, size, act=None, name=None, if select.size is not None: assert select.size == size Layer( - inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( - input, param_attr)] + [select.name], + inputs=[ + Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr) + ] + [select.name], name=name, type=LayerType.SEL_FC_LAYER, size=size, @@ -3409,11 +3664,13 @@ def selective_fc_layer(input, select, size, act=None, name=None, selective_fc_pass_generation=pass_generation, has_selected_colums=has_selected_colums, selective_fc_full_mul_ratio=mul_ratio, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEL_FC_LAYER, list(input) + [select], - activation=act, - size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEL_FC_LAYER, + list(input) + [select], + activation=act, + size=size) @wrap_name_default() @@ -3442,15 +3699,17 @@ def sampling_id_layer(input, name=None, layer_attr=None): name=name, type=LayerType.SAMPLING_ID_LAYER, inputs=[Input(input.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SAMPLING_ID_LAYER, input, - size=l.config.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SAMPLING_ID_LAYER, input, size=l.config.size) @wrap_name_default() @layer_support() -def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0, +def slope_intercept_layer(input, + name=None, + slope=1.0, + intercept=0.0, layer_attr=None): """ This layer for applying a slope and an intercept to the input @@ -3484,16 +3743,14 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0, slope=slope, intercept=intercept, inputs=[Input(input.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SLOPE_INTERCEPT_LAYER, input, - size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SLOPE_INTERCEPT_LAYER, input, size=input.size) @wrap_name_default() @layer_support() -def linear_comb_layer(weights, vectors, size=None, name=None, - layer_attr=None): +def linear_comb_layer(weights, vectors, size=None, name=None, layer_attr=None): """ A layer for weighted sum of vectors takes two inputs. - Input: size of weights is M @@ -3543,7 +3800,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None, if vectors.size is not None and weights.size is not None: assert vectors.size % weights.size == 0 if size is None: - size = vectors.size / weights.size + size = vectors.size / weights.size else: assert size == vectors.size / weights.size Layer( @@ -3551,10 +3808,9 @@ def linear_comb_layer(weights, vectors, size=None, name=None, type=LayerType.LINEAR_COMBINATION_LAYER, size=size, inputs=[Input(weights.name), Input(vectors.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.LINEAR_COMBINATION_LAYER, - [weights, vectors], size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.LINEAR_COMBINATION_LAYER, [weights, vectors], size=size) convex_comb_layer = linear_comb_layer @@ -3626,21 +3882,23 @@ def block_expand_layer(input, if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters - l = Layer(name=name, - inputs=Input(input.name, - block_expand=BlockExpand(channels=num_channels, - block_x=block_x, - block_y=block_y, - stride_x=stride_x, - stride_y=stride_y, - padding_x=padding_x, - padding_y=padding_y)), - type=LayerType.BLOCK_EXPAND, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - - return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input], - size=l.config.size) + l = Layer( + name=name, + inputs=Input( + input.name, + block_expand=BlockExpand( + channels=num_channels, + block_x=block_x, + block_y=block_y, + stride_x=stride_x, + stride_y=stride_y, + padding_x=padding_x, + padding_y=padding_y)), + type=LayerType.BLOCK_EXPAND, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + + return LayerOutput( + name, LayerType.BLOCK_EXPAND, parents=[input], size=l.config.size) @wrap_name_default() @@ -3701,19 +3959,24 @@ def maxout_layer(input, assert input.num_filters is not None num_channels = input.num_filters assert num_channels % groups == 0 - l = Layer(name=name, - inputs=Input(input.name, - maxout=MaxOut(channels=num_channels, - groups=groups)), - type=LayerType.MAXOUT, - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.MAXOUT, parents=[input], - size=l.config.size) + l = Layer( + name=name, + inputs=Input( + input.name, maxout=MaxOut( + channels=num_channels, groups=groups)), + type=LayerType.MAXOUT, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.MAXOUT, parents=[input], size=l.config.size) @wrap_name_default() @layer_support() -def ctc_layer(input, label, size=None, name=None, norm_by_times=False, +def ctc_layer(input, + label, + size=None, + name=None, + norm_by_times=False, layer_attr=None): """ Connectionist Temporal Classification (CTC) is designed for temporal @@ -3769,15 +4032,19 @@ def ctc_layer(input, label, size=None, name=None, norm_by_times=False, size=size, norm_by_times=norm_by_times, inputs=[input.name, label.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size) @wrap_name_default() @wrap_param_attr_default() @layer_support() -def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, +def crf_layer(input, + label, + size=None, + weight=None, + param_attr=None, + name=None, layer_attr=None): """ A layer for calculating the cost of sequential conditional random @@ -3819,8 +4086,7 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, else: assert size == input.size - ipts = [Input(input.name, **param_attr.attr), - Input(label.name)] + ipts = [Input(input.name, **param_attr.attr), Input(label.name)] if weight is not None: ipts.append(Input(weight.name)) @@ -3829,8 +4095,7 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, type=LayerType.CRF_LAYER, size=size, inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input, label] if weight is not None: parents.append(weight) @@ -3843,7 +4108,11 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, @wrap_name_default() @wrap_param_attr_default() @layer_support() -def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, +def crf_decoding_layer(input, + size, + label=None, + param_attr=None, + name=None, layer_attr=None): """ A layer for calculating the decoding sequence of sequential conditional @@ -3880,8 +4149,7 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, type=LayerType.CRF_DECODING_LAYER, size=size, inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input] if label is not None: parents.append(label) @@ -3890,12 +4158,19 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, # classes. return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=1) + @wrap_bias_attr_default(has_bias=True) @wrap_name_default() @layer_support() -def nce_layer(input, label, num_classes, weight=None, - num_neg_samples=10, neg_distribution=None, - name=None, bias_attr=None, layer_attr=None): +def nce_layer(input, + label, + num_classes, + weight=None, + num_neg_samples=10, + neg_distribution=None, + name=None, + bias_attr=None, + layer_attr=None): """ Noise-contrastive estimation. Implements the method in the following paper: @@ -3964,10 +4239,10 @@ def nce_layer(input, label, num_classes, weight=None, num_neg_samples=num_neg_samples, inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.NCE_LAYER, parents=parents, - size=l.config.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.NCE_LAYER, parents=parents, size=l.config.size) + """ following are cost Layers. @@ -3976,7 +4251,13 @@ following are cost Layers. @wrap_name_default() @layer_support() -def rank_cost(left, right, label, weight=None, name=None, coeff=1.0, layer_attr=None): +def rank_cost(left, + right, + label, + weight=None, + name=None, + coeff=1.0, + layer_attr=None): """ A cost Layer for learning to rank using gradient descent. Details can refer to `papers