diff --git a/.clang-format b/.clang-format index 6bbd46d0ff956517991d4faad3f2c026487f412b..9ba433b17362424973626470d930356c2173dd84 100644 --- a/.clang-format +++ b/.clang-format @@ -13,8 +13,6 @@ # The document of clang-format is # http://clang.llvm.org/docs/ClangFormat.html # http://clang.llvm.org/docs/ClangFormatStyleOptions.html -# -# TODO(yuyang18): Add python and other language code style --- Language: Cpp BasedOnStyle: Google @@ -22,8 +20,9 @@ IndentWidth: 2 TabWidth: 2 ContinuationIndentWidth: 4 AccessModifierOffset: -2 # The private/protected/public has no indent in class -PointerAlignment: Left # int* p/int& p, not int *p/int &p Standard: Cpp11 AllowAllParametersOfDeclarationOnNextLine: true +BinPackParameters: false +BinPackArguments: false ... diff --git a/.gitignore b/.gitignore index 65ba217de37c82287829eef105066aba86d69651..ee8489c1d71bd050b9a1d9358a664d2294165292 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ build/ .vscode .idea .project +.cproject .pydevproject +Makefile diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9385943da92bc8c44ca75b267a768ba8ea22bd8b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +- repo: https://github.com/Lucas-C/pre-commit-hooks.git + sha: c25201a00e6b0514370501050cf2a8538ac12270 + hooks: + - id: remove-crlf +- repo: https://github.com/reyoung/mirrors-yapf.git + sha: v0.13.2 + hooks: + - id: yapf +- repo: https://github.com/pre-commit/pre-commit-hooks + sha: 4ef03c4223ad322c7adaa6c6c0efb26b57df3b71 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: check-symlinks + - id: detect-private-key + - id: end-of-file-fixer +# TODO(yuyang): trailing whitespace has some bugs on markdown +# files now, please not add it to pre-commit hook now +# - id: trailing-whitespace +# +# TODO(yuyang): debug-statements not fit for Paddle, because +# not all of our python code is runnable. Some are used for +# documenation +# - id: debug-statements diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000000000000000000000000000000000000..4741fb4f3bbc6681088cf9e960321e7b857a93a8 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,3 @@ +[style] +based_on_style = pep8 +column_limit = 80 diff --git a/demo/image_classification/data/process_cifar.py b/demo/image_classification/data/process_cifar.py index b766118eb00737c7a196ed85850b3cebd690b0d0..b235010e4ece377beffaaa1b9247a77d7a96b712 100644 --- a/demo/image_classification/data/process_cifar.py +++ b/demo/image_classification/data/process_cifar.py @@ -16,7 +16,6 @@ import numpy as np import sys import os import PIL.Image as Image - """ Usage: python process_cifar input_dir output_dir """ @@ -30,6 +29,7 @@ def mkdir_not_exist(path): if not os.path.exists(path): os.mkdir(path) + def create_dir_structure(output_dir): """ Create the directory structure for the directory. @@ -39,8 +39,8 @@ def create_dir_structure(output_dir): mkdir_not_exist(os.path.join(output_dir, "train")) mkdir_not_exist(os.path.join(output_dir, "test")) -def convert_batch(batch_path, label_set, label_map, - output_dir, data_split): + +def convert_batch(batch_path, label_set, label_map, output_dir, data_split): """ Convert CIFAR batch to the structure of Paddle format. batch_path: the batch to be converted. @@ -67,11 +67,23 @@ if __name__ == '__main__': output_dir = sys.argv[2] num_batch = 5 create_dir_structure(output_dir) - label_map = {0: "airplane", 1: "automobile", 2: "bird", 3: "cat", 4: "deer", - 5: "dog", 6: "frog", 7: "horse", 8: "ship", 9: "truck"} + label_map = { + 0: "airplane", + 1: "automobile", + 2: "bird", + 3: "cat", + 4: "deer", + 5: "dog", + 6: "frog", + 7: "horse", + 8: "ship", + 9: "truck" + } labels = {} for i in range(1, num_batch + 1): - convert_batch(os.path.join(input_dir, "data_batch_%d" % i), labels, - label_map, output_dir, "train") - convert_batch(os.path.join(input_dir, "test_batch"), {}, - label_map, output_dir, "test") \ No newline at end of file + convert_batch( + os.path.join(input_dir, "data_batch_%d" % i), labels, label_map, + output_dir, "train") + convert_batch( + os.path.join(input_dir, "test_batch"), {}, label_map, output_dir, + "test") diff --git a/demo/image_classification/image_provider.py b/demo/image_classification/image_provider.py index 305efbcdc6bb11f1dac65cc3af82fb997db97f27..28bf1bb02c1f08b2e8ec9acd38f0a8594b05ab66 100644 --- a/demo/image_classification/image_provider.py +++ b/demo/image_classification/image_provider.py @@ -46,14 +46,14 @@ def hook(settings, img_size, mean_img_size, num_classes, color, meta, use_jpeg, settings.img_mean = image_util.load_meta(settings.meta_path, settings.mean_img_size, - settings.img_size, - settings.color) + settings.img_size, settings.color) settings.logger.info('Image size: %s', settings.img_size) settings.logger.info('Meta path: %s', settings.meta_path) settings.input_types = [ dense_vector(settings.img_raw_size), # image feature - integer_value(settings.num_classes)] # labels + integer_value(settings.num_classes) + ] # labels settings.logger.info('DataProvider Initialization finished') @@ -79,8 +79,8 @@ def processData(settings, file_list): img = image_util.decode_jpeg(data['images'][i]) else: img = data['images'][i] - img_feat = image_util.preprocess_img(img, settings.img_mean, - settings.img_size, settings.is_train, - settings.color) + img_feat = image_util.preprocess_img( + img, settings.img_mean, settings.img_size, + settings.is_train, settings.color) label = data['labels'][i] yield img_feat.astype('float32'), int(label) diff --git a/demo/image_classification/image_util.py b/demo/image_classification/image_util.py index c545d16aafbc741bce25f9469e7f67de5b88fa8c..b5c6431c06f77cef5c31ca844a8427eebaea2fce 100644 --- a/demo/image_classification/image_util.py +++ b/demo/image_classification/image_util.py @@ -16,17 +16,20 @@ import numpy as np from PIL import Image from cStringIO import StringIO + def resize_image(img, target_size): """ Resize an image so that the shorter edge has length target_size. img: the input image to be resized. target_size: the target resized image size. """ - percent = (target_size/float(min(img.size[0], img.size[1]))) - resized_size = int(round(img.size[0] * percent)), int(round(img.size[1] * percent)) + percent = (target_size / float(min(img.size[0], img.size[1]))) + resized_size = int(round(img.size[0] * percent)), int( + round(img.size[1] * percent)) img = img.resize(resized_size, Image.ANTIALIAS) return img + def flip(im): """ Return the flipped image. @@ -38,6 +41,7 @@ def flip(im): else: return im[:, ::-1] + def crop_img(im, inner_size, color=True, test=True): """ Return cropped image. @@ -50,20 +54,22 @@ def crop_img(im, inner_size, color=True, test=True): If True, crop the center of images. """ if color: - height, width = max(inner_size, im.shape[1]), max(inner_size, im.shape[2]) + height, width = max(inner_size, im.shape[1]), max(inner_size, + im.shape[2]) padded_im = np.zeros((3, height, width)) startY = (height - im.shape[1]) / 2 startX = (width - im.shape[2]) / 2 endY, endX = startY + im.shape[1], startX + im.shape[2] - padded_im[:, startY: endY, startX: endX] = im + padded_im[:, startY:endY, startX:endX] = im else: im = im.astype('float32') - height, width = max(inner_size, im.shape[0]), max(inner_size, im.shape[1]) + height, width = max(inner_size, im.shape[0]), max(inner_size, + im.shape[1]) padded_im = np.zeros((height, width)) startY = (height - im.shape[0]) / 2 startX = (width - im.shape[1]) / 2 endY, endX = startY + im.shape[0], startX + im.shape[1] - padded_im[startY: endY, startX: endX] = im + padded_im[startY:endY, startX:endX] = im if test: startY = (height - inner_size) / 2 startX = (width - inner_size) / 2 @@ -72,19 +78,21 @@ def crop_img(im, inner_size, color=True, test=True): startX = np.random.randint(0, width - inner_size + 1) endY, endX = startY + inner_size, startX + inner_size if color: - pic = padded_im[:, startY: endY, startX: endX] + pic = padded_im[:, startY:endY, startX:endX] else: - pic = padded_im[startY: endY, startX: endX] + pic = padded_im[startY:endY, startX:endX] if (not test) and (np.random.randint(2) == 0): pic = flip(pic) return pic + def decode_jpeg(jpeg_string): np_array = np.array(Image.open(StringIO(jpeg_string))) if len(np_array.shape) == 3: np_array = np.transpose(np_array, (2, 0, 1)) return np_array + def preprocess_img(im, img_mean, crop_size, is_train, color=True): """ Does data augmentation for images. @@ -99,6 +107,7 @@ def preprocess_img(im, img_mean, crop_size, is_train, color=True): pic -= img_mean return pic.flatten() + def load_meta(meta_path, mean_img_size, crop_size, color=True): """ Return the loaded meta file. @@ -109,17 +118,18 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True): mean = np.load(meta_path)['data_mean'] border = (mean_img_size - crop_size) / 2 if color: - assert(mean_img_size * mean_img_size * 3 == mean.shape[0]) + assert (mean_img_size * mean_img_size * 3 == mean.shape[0]) mean = mean.reshape(3, mean_img_size, mean_img_size) - mean = mean[:, border: border + crop_size, - border: border + crop_size].astype('float32') + mean = mean[:, border:border + crop_size, border:border + + crop_size].astype('float32') else: - assert(mean_img_size * mean_img_size == mean.shape[0]) + assert (mean_img_size * mean_img_size == mean.shape[0]) mean = mean.reshape(mean_img_size, mean_img_size) - mean = mean[border: border + crop_size, - border: border + crop_size].astype('float32') + mean = mean[border:border + crop_size, border:border + + crop_size].astype('float32') return mean + def load_image(img_path, is_color=True): """ Load image and return. @@ -130,6 +140,7 @@ def load_image(img_path, is_color=True): img.load() return img + def oversample(img, crop_dims): """ image : iterable of (H x W x K) ndarrays @@ -152,50 +163,53 @@ def oversample(img, crop_dims): for j in w_indices: crops_ix[curr] = (i, j, i + crop_dims[0], j + crop_dims[1]) curr += 1 - crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate([ - -crop_dims / 2.0, - crop_dims / 2.0 - ]) + crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate( + [-crop_dims / 2.0, crop_dims / 2.0]) crops_ix = np.tile(crops_ix, (2, 1)) # Extract crops - crops = np.empty((10 * len(img), crop_dims[0], crop_dims[1], - im_shape[-1]), dtype=np.float32) + crops = np.empty( + (10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), + dtype=np.float32) ix = 0 for im in img: for crop in crops_ix: crops[ix] = im[crop[0]:crop[2], crop[1]:crop[3], :] ix += 1 - crops[ix-5:ix] = crops[ix-5:ix, :, ::-1, :] # flip for mirrors + crops[ix - 5:ix] = crops[ix - 5:ix, :, ::-1, :] # flip for mirrors return crops + class ImageTransformer: - def __init__(self, transpose = None, - channel_swap = None, mean = None, is_color = True): + def __init__(self, + transpose=None, + channel_swap=None, + mean=None, + is_color=True): self.transpose = transpose self.channel_swap = None self.mean = None - self.is_color = is_color + self.is_color = is_color - def set_transpose(self, order): + def set_transpose(self, order): if self.is_color: - assert 3 == len(order) + assert 3 == len(order) self.transpose = order - def set_channel_swap(self, order): + def set_channel_swap(self, order): if self.is_color: - assert 3 == len(order) + assert 3 == len(order) self.channel_swap = order def set_mean(self, mean): # mean value, may be one value per channel if mean.ndim == 1: - mean = mean[:, np.newaxis, np.newaxis] - else: + mean = mean[:, np.newaxis, np.newaxis] + else: # elementwise mean if self.is_color: assert len(mean.shape) == 3 - self.mean = mean + self.mean = mean def transformer(self, data): if self.transpose is not None: diff --git a/demo/image_classification/prediction.py b/demo/image_classification/prediction.py index 5d9e93265867389ca6d2aa26e48fcfa08561e6ae..6a47bd5851c99635dd7d3f1d5df67dd081ca4584 100755 --- a/demo/image_classification/prediction.py +++ b/demo/image_classification/prediction.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os,sys +import os, sys import numpy as np import logging from PIL import Image @@ -24,9 +24,11 @@ from py_paddle import swig_paddle, DataProviderConverter from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + class ImageClassifier(): def __init__(self, train_conf, @@ -58,18 +60,19 @@ class ImageClassifier(): self.oversample = oversample self.is_color = is_color - self.transformer = image_util.ImageTransformer(is_color = is_color) - self.transformer.set_transpose((2,0,1)) + self.transformer = image_util.ImageTransformer(is_color=is_color) + self.transformer.set_transpose((2, 0, 1)) self.mean_file = mean_file mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) - self.transformer.set_mean(mean) # mean pixel + self.transformer.set_mean(mean) # mean pixel gpu = 1 if use_gpu else 0 conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (gpu) conf = parse_config(train_conf, conf_args) swig_paddle.initPaddle("--use_gpu=%d" % (gpu)) - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) self.network.loadParameters(self.model_dir) @@ -90,14 +93,14 @@ class ImageClassifier(): # image_util.resize_image: short side is self.resize_dim image = image_util.resize_image(image, self.resize_dim) image = np.array(image) - input = np.zeros((1, image.shape[0], image.shape[1], 3), - dtype=np.float32) + input = np.zeros( + (1, image.shape[0], image.shape[1], 3), dtype=np.float32) input[0] = image.astype(np.float32) input = image_util.oversample(input, self.crop_dims) else: image = image.resize(self.crop_dims, Image.ANTIALIAS) - input = np.zeros((1, self.crop_dims[0], self.crop_dims[1], 3), - dtype=np.float32) + input = np.zeros( + (1, self.crop_dims[0], self.crop_dims[1], 3), dtype=np.float32) input[0] = np.array(image).astype(np.float32) data_in = [] @@ -133,22 +136,24 @@ class ImageClassifier(): lab = np.argsort(-prob) logging.info("Label of %s is: %d", image, lab[0]) + if __name__ == '__main__': - image_size=32 - crop_size=32 - multi_crop=True - config="vgg_16_cifar.py" - output_layer="__fc_layer_1__" - mean_path="data/cifar-out/batches/batches.meta" - model_path=sys.argv[1] - image=sys.argv[2] - use_gpu=bool(int(sys.argv[3])) - - obj = ImageClassifier(train_conf=config, - model_dir=model_path, - resize_dim=image_size, - crop_dim=crop_size, - mean_file=mean_path, - use_gpu=use_gpu, - oversample=multi_crop) + image_size = 32 + crop_size = 32 + multi_crop = True + config = "vgg_16_cifar.py" + output_layer = "__fc_layer_1__" + mean_path = "data/cifar-out/batches/batches.meta" + model_path = sys.argv[1] + image = sys.argv[2] + use_gpu = bool(int(sys.argv[3])) + + obj = ImageClassifier( + train_conf=config, + model_dir=model_path, + resize_dim=image_size, + crop_dim=crop_size, + mean_file=mean_path, + use_gpu=use_gpu, + oversample=multi_crop) obj.predict(image, output_layer) diff --git a/demo/image_classification/preprocess.py b/demo/image_classification/preprocess.py index fe7ea19bf02776629dff0f64f5b671dc457eae64..10b9c1691b5e51273c73a975545cd36f3822e901 100755 --- a/demo/image_classification/preprocess.py +++ b/demo/image_classification/preprocess.py @@ -19,24 +19,36 @@ from optparse import OptionParser def option_parser(): parser = OptionParser(usage="usage: python preprcoess.py "\ "-i data_dir [options]") - parser.add_option("-i", "--input", action="store", - dest="input", help="Input data directory.") - parser.add_option("-s", "--size", action="store", - dest="size", help="Processed image size.") - parser.add_option("-c", "--color", action="store", - dest="color", help="whether to use color images.") + parser.add_option( + "-i", + "--input", + action="store", + dest="input", + help="Input data directory.") + parser.add_option( + "-s", + "--size", + action="store", + dest="size", + help="Processed image size.") + parser.add_option( + "-c", + "--color", + action="store", + dest="color", + help="whether to use color images.") return parser.parse_args() + if __name__ == '__main__': - options, args = option_parser() - data_dir = options.input - processed_image_size = int(options.size) - color = options.color == "1" - data_creator = ImageClassificationDatasetCreater(data_dir, - processed_image_size, - color) - data_creator.train_list_name = "train.txt" - data_creator.test_list_name = "test.txt" - data_creator.num_per_batch = 1000 - data_creator.overwrite = True - data_creator.create_batches() + options, args = option_parser() + data_dir = options.input + processed_image_size = int(options.size) + color = options.color == "1" + data_creator = ImageClassificationDatasetCreater( + data_dir, processed_image_size, color) + data_creator.train_list_name = "train.txt" + data_creator.test_list_name = "test.txt" + data_creator.num_per_batch = 1000 + data_creator.overwrite = True + data_creator.create_batches() diff --git a/demo/image_classification/vgg_16_cifar.py b/demo/image_classification/vgg_16_cifar.py index edd6988c48acd6b554e09b721c37b291e21f46eb..58ceff5fc2f46cac9997b6d8af2b0db0c43e0c75 100755 --- a/demo/image_classification/vgg_16_cifar.py +++ b/demo/image_classification/vgg_16_cifar.py @@ -18,36 +18,38 @@ is_predict = get_config_arg("is_predict", bool, False) ####################Data Configuration ################## if not is_predict: - data_dir='data/cifar-out/batches/' - meta_path=data_dir+'batches.meta' - - args = {'meta':meta_path,'mean_img_size': 32, - 'img_size': 32,'num_classes': 10, - 'use_jpeg': 1,'color': "color"} - - define_py_data_sources2(train_list="train.list", - test_list="train.list", - module='image_provider', - obj='processData', - args=args) + data_dir = 'data/cifar-out/batches/' + meta_path = data_dir + 'batches.meta' + + args = { + 'meta': meta_path, + 'mean_img_size': 32, + 'img_size': 32, + 'num_classes': 10, + 'use_jpeg': 1, + 'color': "color" + } + + define_py_data_sources2( + train_list="train.list", + test_list="train.list", + module='image_provider', + obj='processData', + args=args) ######################Algorithm Configuration ############# settings( - batch_size = 128, - learning_rate = 0.1 / 128.0, - learning_method = MomentumOptimizer(0.9), - regularization = L2Regularization(0.0005 * 128) -) + batch_size=128, + learning_rate=0.1 / 128.0, + learning_method=MomentumOptimizer(0.9), + regularization=L2Regularization(0.0005 * 128)) #######################Network Configuration ############# -data_size=3*32*32 -label_size=10 -img = data_layer(name='image', - size=data_size) +data_size = 3 * 32 * 32 +label_size = 10 +img = data_layer(name='image', size=data_size) # small_vgg is predefined in trainer_config_helpers.networks -predict = small_vgg(input_image=img, - num_channels=3, - num_classes=label_size) +predict = small_vgg(input_image=img, num_channels=3, num_classes=label_size) if not is_predict: lbl = data_layer(name="label", size=label_size) diff --git a/demo/introduction/README.md b/demo/introduction/README.md index bebf1d090d98691199ede55736dfe5b964a8fd42..0614a7afe645677ef0b65a17ea05f1dcfa45214f 100644 --- a/demo/introduction/README.md +++ b/demo/introduction/README.md @@ -1,4 +1,3 @@ This folder contains scripts used in PaddlePaddle introduction. - use `bash train.sh` to train a simple linear regression model - use `python evaluate_model.py` to read model parameters. You can see that `w` and `b` are very close to [2, 0.3]. - diff --git a/demo/introduction/dataprovider.py b/demo/introduction/dataprovider.py index be8c0bc89156cf843d9b08276b52f92a4d8c9706..8515022e18dc6bbf055e6db3121568acf1df1c55 100644 --- a/demo/introduction/dataprovider.py +++ b/demo/introduction/dataprovider.py @@ -15,10 +15,10 @@ from paddle.trainer.PyDataProvider2 import * import random + # define data types of input: 2 real numbers -@provider(input_types=[dense_vector(1), dense_vector(1)],use_seq=False) +@provider(input_types=[dense_vector(1), dense_vector(1)], use_seq=False) def process(settings, input_file): for i in xrange(2000): x = random.random() - yield [x], [2*x+0.3] - + yield [x], [2 * x + 0.3] diff --git a/demo/introduction/evaluate_model.py b/demo/introduction/evaluate_model.py index 8cfb843c42105757b0f63c4a00d034b47a37a0bb..ca4a1872731abde90e72cb167929b3d9e2e1ebf4 100755 --- a/demo/introduction/evaluate_model.py +++ b/demo/introduction/evaluate_model.py @@ -23,14 +23,17 @@ Usage: import numpy as np import os + def load(file_name): with open(file_name, 'rb') as f: - f.read(16) # skip header for float type. + f.read(16) # skip header for float type. return np.fromfile(f, dtype=np.float32) + def main(): print 'w=%.6f, b=%.6f from pass 29' % (load('output/pass-00029/w'), - load('output/pass-00029/b')) + load('output/pass-00029/b')) + if __name__ == '__main__': main() diff --git a/demo/introduction/trainer_config.py b/demo/introduction/trainer_config.py index 3e3df5583282a4335ddea7b1cb30a84052d0adca..7c838c1a8f5b3cb6ac732197c85cd7c728eb013f 100644 --- a/demo/introduction/trainer_config.py +++ b/demo/introduction/trainer_config.py @@ -16,9 +16,14 @@ from paddle.trainer_config_helpers import * # 1. read data. Suppose you saved above python code as dataprovider.py data_file = 'empty.list' -with open(data_file, 'w') as f: f.writelines(' ') -define_py_data_sources2(train_list=data_file, test_list=None, - module='dataprovider', obj='process',args={}) +with open(data_file, 'w') as f: + f.writelines(' ') +define_py_data_sources2( + train_list=data_file, + test_list=None, + module='dataprovider', + obj='process', + args={}) # 2. learning algorithm settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) @@ -26,7 +31,11 @@ settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) # 3. Network configuration x = data_layer(name='x', size=1) y = data_layer(name='y', size=1) -y_predict = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) +y_predict = fc_layer( + input=x, + param_attr=ParamAttr(name='w'), + size=1, + act=LinearActivation(), + bias_attr=ParamAttr(name='b')) cost = regression_cost(input=y_predict, label=y) outputs(cost) - diff --git a/demo/mnist/data/generate_list.py b/demo/mnist/data/generate_list.py index 1b929048b4d82b5e9d80585b6d0180f2e92200ce..d880721f94c68bbbc1740f82872462efdb368fa2 100644 --- a/demo/mnist/data/generate_list.py +++ b/demo/mnist/data/generate_list.py @@ -13,9 +13,9 @@ # limitations under the License. o = open("./" + "train.list", "w") -o.write("./data/raw_data/train" +"\n") +o.write("./data/raw_data/train" + "\n") o.close() o = open("./" + "test.list", "w") -o.write("./data/raw_data/t10k" +"\n") -o.close() \ No newline at end of file +o.write("./data/raw_data/t10k" + "\n") +o.close() diff --git a/demo/mnist/data/get_mnist_data.sh b/demo/mnist/data/get_mnist_data.sh index 9099b5ab6fb85d86d346a7ad819538fbd013c6ff..5a2e34026d4fe7f8315d4f5453bec7c4ee4f6885 100755 --- a/demo/mnist/data/get_mnist_data.sh +++ b/demo/mnist/data/get_mnist_data.sh @@ -19,4 +19,3 @@ done cd $DIR rm -f *.list python generate_list.py - diff --git a/demo/mnist/mnist_provider.py b/demo/mnist/mnist_provider.py index 32af29730a7365df1a98fe54a2edf8850ee93e8d..6df4676da3bdc2e6949cc911fa3720cb51ddc568 100644 --- a/demo/mnist/mnist_provider.py +++ b/demo/mnist/mnist_provider.py @@ -2,10 +2,9 @@ from paddle.trainer.PyDataProvider2 import * # Define a py data provider -@provider(input_types={ - 'pixel': dense_vector(28 * 28), - 'label': integer_value(10) -}) +@provider( + input_types={'pixel': dense_vector(28 * 28), + 'label': integer_value(10)}) def process(settings, filename): # settings is not used currently. imgf = filename + "-images-idx3-ubyte" labelf = filename + "-labels-idx1-ubyte" diff --git a/demo/mnist/vgg_16_mnist.py b/demo/mnist/vgg_16_mnist.py index 45a45bb061aa781231a944bb82ebfbc6b0dc9618..f9e89bc588abacd98a8f5fc82a00fae6bb2de10e 100644 --- a/demo/mnist/vgg_16_mnist.py +++ b/demo/mnist/vgg_16_mnist.py @@ -18,32 +18,29 @@ is_predict = get_config_arg("is_predict", bool, False) ####################Data Configuration ################## - if not is_predict: - data_dir='./data/' - define_py_data_sources2(train_list= data_dir + 'train.list', - test_list= data_dir + 'test.list', - module='mnist_provider', - obj='process') + data_dir = './data/' + define_py_data_sources2( + train_list=data_dir + 'train.list', + test_list=data_dir + 'test.list', + module='mnist_provider', + obj='process') ######################Algorithm Configuration ############# settings( - batch_size = 128, - learning_rate = 0.1 / 128.0, - learning_method = MomentumOptimizer(0.9), - regularization = L2Regularization(0.0005 * 128) -) + batch_size=128, + learning_rate=0.1 / 128.0, + learning_method=MomentumOptimizer(0.9), + regularization=L2Regularization(0.0005 * 128)) #######################Network Configuration ############# -data_size=1*28*28 -label_size=10 +data_size = 1 * 28 * 28 +label_size = 10 img = data_layer(name='pixel', size=data_size) # small_vgg is predined in trainer_config_helpers.network -predict = small_vgg(input_image=img, - num_channels=1, - num_classes=label_size) +predict = small_vgg(input_image=img, num_channels=1, num_classes=label_size) if not is_predict: lbl = data_layer(name="label", size=label_size) diff --git a/demo/model_zoo/embedding/extract_para.py b/demo/model_zoo/embedding/extract_para.py index 17067792fc38d0d25bd28dc35bfb1b88ad5020cd..47e06fae9caa9c3d9e0d6eb2e3f6633a776c5b1d 100755 --- a/demo/model_zoo/embedding/extract_para.py +++ b/demo/model_zoo/embedding/extract_para.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python extract_para.py --preModel PREMODEL --preDict PREDICT \ @@ -29,6 +28,7 @@ Options: from optparse import OptionParser import struct + def get_row_index(preDict, usrDict): """ Get the row positions for all words in user dictionary from pre-trained dictionary. @@ -47,7 +47,9 @@ def get_row_index(preDict, usrDict): pos.append(index[word]) return pos -def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, paraDim): + +def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, + paraDim): """ Extract desired parameters from a pretrained embedding model based on user dictionary """ @@ -70,6 +72,7 @@ def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, paraDim) print "extract parameters finish, total", len(rowIndex), "lines" fi.close() + def main(): """ Main entry for running paraconvert.py @@ -78,19 +81,33 @@ def main(): "python %prog --preModel PREMODEL --preDict PREDICT" \ " --usrModel USRMODEL --usrDict USRDICT -d DIM" parser = OptionParser(usage) - parser.add_option("--preModel", action="store", dest="preModel", - help="the name of pretrained embedding model") - parser.add_option("--preDict", action="store", dest="preDict", - help="the name of pretrained dictionary") - parser.add_option("--usrModel", action="store", dest="usrModel", - help="the name of output usr embedding model") - parser.add_option("--usrDict", action="store", dest="usrDict", - help="the name of user specified dictionary") - parser.add_option("-d", action="store", dest="dim", - help="dimension of parameter") + parser.add_option( + "--preModel", + action="store", + dest="preModel", + help="the name of pretrained embedding model") + parser.add_option( + "--preDict", + action="store", + dest="preDict", + help="the name of pretrained dictionary") + parser.add_option( + "--usrModel", + action="store", + dest="usrModel", + help="the name of output usr embedding model") + parser.add_option( + "--usrDict", + action="store", + dest="usrDict", + help="the name of user specified dictionary") + parser.add_option( + "-d", action="store", dest="dim", help="dimension of parameter") (options, args) = parser.parse_args() - extract_parameters_by_usrDict(options.preModel, options.preDict, - options.usrModel, options.usrDict, int(options.dim)) + extract_parameters_by_usrDict(options.preModel, options.preDict, + options.usrModel, options.usrDict, + int(options.dim)) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/embedding/paraconvert.py b/demo/model_zoo/embedding/paraconvert.py index 523412303617a38035392e4bb99f8ce119be8ac8..54155eff8e26b16ff5303d8d279e81b4bf8a90f4 100755 --- a/demo/model_zoo/embedding/paraconvert.py +++ b/demo/model_zoo/embedding/paraconvert.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM @@ -29,6 +28,7 @@ Options: from optparse import OptionParser import struct + def binary2text(input, output, paraDim): """ Convert a binary parameter file of embedding model to be a text file. @@ -76,12 +76,13 @@ def binary2text(input, output, paraDim): fo.close() print "binary2text finish, total", line, "lines" + def get_para_count(input): """ Compute the total number of embedding parameters in input text file. input: the name of input text file """ - numRows = 1 + numRows = 1 paraDim = 0 with open(input) as f: line = f.readline() @@ -90,6 +91,7 @@ def get_para_count(input): numRows += 1 return numRows * paraDim + def text2binary(input, output, paddle_head=True): """ Convert a text parameter file of embedding model to be a binary file. @@ -123,6 +125,7 @@ def text2binary(input, output, paddle_head=True): fo.close() print "text2binary finish, total", count, "lines" + def main(): """ Main entry for running paraconvert.py @@ -131,21 +134,26 @@ def main(): "python %prog --b2t -i INPUT -o OUTPUT -d DIM \n" \ "python %prog --t2b -i INPUT -o OUTPUT" parser = OptionParser(usage) - parser.add_option("--b2t", action="store_true", - help="convert parameter file of embedding model from binary to text") - parser.add_option("--t2b", action="store_true", - help="convert parameter file of embedding model from text to binary") - parser.add_option("-i", action="store", dest="input", - help="input parameter file name") - parser.add_option("-o", action="store", dest="output", - help="output parameter file name") - parser.add_option("-d", action="store", dest="dim", - help="dimension of parameter") + parser.add_option( + "--b2t", + action="store_true", + help="convert parameter file of embedding model from binary to text") + parser.add_option( + "--t2b", + action="store_true", + help="convert parameter file of embedding model from text to binary") + parser.add_option( + "-i", action="store", dest="input", help="input parameter file name") + parser.add_option( + "-o", action="store", dest="output", help="output parameter file name") + parser.add_option( + "-d", action="store", dest="dim", help="dimension of parameter") (options, args) = parser.parse_args() if options.b2t: binary2text(options.input, options.output, options.dim) if options.t2b: text2binary(options.input, options.output) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/resnet/classify.py b/demo/model_zoo/resnet/classify.py index 06d471722f8059804a59e6823bebccff85a8d542..7855126edcfec20de251e5bc08c08c7aab8f7a8e 100755 --- a/demo/model_zoo/resnet/classify.py +++ b/demo/model_zoo/resnet/classify.py @@ -26,16 +26,22 @@ from py_paddle import swig_paddle, DataProviderConverter from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + class ImageClassifier(): - def __init__(self, train_conf, model_dir=None, - resize_dim=256, crop_dim=224, + def __init__(self, + train_conf, + model_dir=None, + resize_dim=256, + crop_dim=224, use_gpu=True, mean_file=None, output_layer=None, - oversample=False, is_color=True): + oversample=False, + is_color=True): """ train_conf: network configure. model_dir: string, directory of model. @@ -62,24 +68,25 @@ class ImageClassifier(): assert isinstance(self.output_layer, basestring) self.output_layer = self.output_layer.split(",") - self.transformer = image_util.ImageTransformer(is_color = is_color) - self.transformer.set_transpose((2,0,1)) - self.transformer.set_channel_swap((2,1,0)) + self.transformer = image_util.ImageTransformer(is_color=is_color) + self.transformer.set_transpose((2, 0, 1)) + self.transformer.set_channel_swap((2, 1, 0)) self.mean_file = mean_file if self.mean_file is not None: mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) - self.transformer.set_mean(mean) # mean pixel + self.transformer.set_mean(mean) # mean pixel else: # if you use three mean value, set like: # this three mean value is calculated from ImageNet. - self.transformer.set_mean(np.array([103.939,116.779,123.68])) + self.transformer.set_mean(np.array([103.939, 116.779, 123.68])) conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (int(use_gpu)) conf = parse_config(train_conf, conf_args) swig_paddle.initPaddle("--use_gpu=%d" % (int(use_gpu))) - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) self.network.loadParameters(self.model_dir) @@ -105,14 +112,14 @@ class ImageClassifier(): # image_util.resize_image: short side is self.resize_dim image = image_util.resize_image(image, self.resize_dim) image = np.array(image) - input = np.zeros((1, image.shape[0], image.shape[1], 3), - dtype=np.float32) + input = np.zeros( + (1, image.shape[0], image.shape[1], 3), dtype=np.float32) input[0] = image.astype(np.float32) input = image_util.oversample(input, self.crop_dims) else: image = image.resize(self.crop_dims, Image.ANTIALIAS) - input = np.zeros((1, self.crop_dims[0], self.crop_dims[1], 3), - dtype=np.float32) + input = np.zeros( + (1, self.crop_dims[0], self.crop_dims[1], 3), dtype=np.float32) input[0] = np.array(image).astype(np.float32) data_in = [] @@ -172,7 +179,7 @@ class ImageClassifier(): logging.info("Label of %s is: %d", image, lab[0]) return results - def extract(self, data_file, output_dir, batch_size = 10000): + def extract(self, data_file, output_dir, batch_size=10000): """ extract and save features of output layers, which are specify in Outputs() in network configure. @@ -197,7 +204,7 @@ class ImageClassifier(): image_feature[file_name] = feature sample_num += 1 if sample_num == batch_size: - batch_name = os.path.join(output_dir, 'batch_%d' %(batch_num)) + batch_name = os.path.join(output_dir, 'batch_%d' % (batch_num)) self.save_file(image_feature, batch_name) logging.info('Finish batch %d', batch_num) batch_num += 1 @@ -206,7 +213,7 @@ class ImageClassifier(): if idx % 1000 == 0: logging.info('%d/%d, %s', idx, len(image_files), file_name) if sample_num > 0: - batch_name = os.path.join(output_dir, 'batch_%d' %(batch_num)) + batch_name = os.path.join(output_dir, 'batch_%d' % (batch_num)) self.save_file(image_feature, batch_name) logging.info('Finish batch %d', batch_num) logging.info('Done: make image feature batch') @@ -215,38 +222,64 @@ class ImageClassifier(): of = open(file, 'wb') cPickle.dump(data, of, protocol=cPickle.HIGHEST_PROTOCOL) + def option_parser(): """ Main entry for predciting """ usage = "%prog -c config -i data_list -w model_dir [options]" parser = OptionParser(usage="usage: %s" % usage) - parser.add_option("-j", "--job", - action="store", dest="job_type", - help="job type: predict, extract\ + parser.add_option( + "-j", + "--job", + action="store", + dest="job_type", + help="job type: predict, extract\ predict: predicting,\ extract: extract features") - parser.add_option("-c", "--conf", - action="store", dest="train_conf", - help="network config") - parser.add_option("-i", "--data", - action="store", dest="data_file", - help="image list") - parser.add_option("-w", "--model", - action="store", dest="model_path", - default=None, help="model path") - parser.add_option("-g", "--use_gpu", action="store", - dest="use_gpu", default=True, - help="Whether to use gpu mode.") - parser.add_option("-o", "--output_dir", - action="store", dest="output_dir", - default="output", help="output path") - parser.add_option("-m", "--mean", action="store", - dest="mean", default=None, - help="mean file.") - parser.add_option("-p", "--multi_crop", action="store_true", - dest="multi_crop", default=False, - help="Wether to use multiple crops on image.") + parser.add_option( + "-c", + "--conf", + action="store", + dest="train_conf", + help="network config") + parser.add_option( + "-i", "--data", action="store", dest="data_file", help="image list") + parser.add_option( + "-w", + "--model", + action="store", + dest="model_path", + default=None, + help="model path") + parser.add_option( + "-g", + "--use_gpu", + action="store", + dest="use_gpu", + default=True, + help="Whether to use gpu mode.") + parser.add_option( + "-o", + "--output_dir", + action="store", + dest="output_dir", + default="output", + help="output path") + parser.add_option( + "-m", + "--mean", + action="store", + dest="mean", + default=None, + help="mean file.") + parser.add_option( + "-p", + "--multi_crop", + action="store_true", + dest="multi_crop", + default=False, + help="Wether to use multiple crops on image.") parser.add_option("-l", "--output_layer", action="store", dest="output_layer", default=None, help="--job=extract, specify layers to extract "\ @@ -254,24 +287,26 @@ def option_parser(): "classification probability, output in resnet.py.") return parser.parse_args() + def main(): """ 1. parse input arguments. 2. predicting or extract features according job type. """ options, args = option_parser() - obj = ImageClassifier(options.train_conf, - options.model_path, - use_gpu=options.use_gpu, - mean_file=options.mean, - output_layer=options.output_layer, - oversample=options.multi_crop) + obj = ImageClassifier( + options.train_conf, + options.model_path, + use_gpu=options.use_gpu, + mean_file=options.mean, + output_layer=options.output_layer, + oversample=options.multi_crop) if options.job_type == "predict": obj.predict(options.data_file) elif options.job_type == "extract": - obj.extract(options.data_file, - options.output_dir) + obj.extract(options.data_file, options.output_dir) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/resnet/example/__init__.py b/demo/model_zoo/resnet/example/__init__.py index 7f9e87eee6037666b86420fba194624859d356b3..c90af2ee000d46a032984ee23559e7e99b49ddad 100644 --- a/demo/model_zoo/resnet/example/__init__.py +++ b/demo/model_zoo/resnet/example/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/demo/model_zoo/resnet/example/image_list_provider.py b/demo/model_zoo/resnet/example/image_list_provider.py index ee457e1fffc7ed8629dc6bde63a8047818c0ff9d..9e415f76a53326c5809b7a8c508701c519ab443b 100644 --- a/demo/model_zoo/resnet/example/image_list_provider.py +++ b/demo/model_zoo/resnet/example/image_list_provider.py @@ -16,8 +16,7 @@ from paddle.utils.image_util import * from paddle.trainer.PyDataProvider2 import * -def hook(settings, image_size, crop_size, color, file_list, - is_train, **kwargs): +def hook(settings, image_size, crop_size, color, file_list, is_train, **kwargs): """ Description: Init with a list of data file file_list is the name list of input files. @@ -58,7 +57,7 @@ def hook(settings, image_size, crop_size, color, file_list, sz = settings.crop_size * settings.crop_size settings.img_mean = np.zeros(sz * 3, dtype=np.single) for idx, value in enumerate(settings.mean_value): - settings.img_mean[idx * sz: (idx + 1) * sz] = value + settings.img_mean[idx * sz:(idx + 1) * sz] = value settings.img_mean = settings.img_mean.reshape(3, settings.crop_size, settings.crop_size) @@ -69,7 +68,8 @@ def hook(settings, image_size, crop_size, color, file_list, settings.input_types = [ dense_vector(settings.img_input_size), # image feature - integer_value(1)] # labels + integer_value(1) + ] # labels settings.logger.info('Image short side: %s', settings.img_size) settings.logger.info('Crop size: %s', settings.crop_size) @@ -97,9 +97,6 @@ def processData(settings, file_list): # swap channel if settings.is_swap_channel: img = img[settings.swap_channel, :, :] - img_feat = preprocess_img(img, - settings.img_mean, - settings.crop_size, - settings.is_train, - settings.color) + img_feat = preprocess_img(img, settings.img_mean, settings.crop_size, + settings.is_train, settings.color) yield img_feat.tolist(), int(lab.strip()) diff --git a/demo/model_zoo/resnet/load_feature.py b/demo/model_zoo/resnet/load_feature.py index ee4930b7a17f7f21ceeba8db253eed64416ebf10..b0948b75fd0ac9a3fa89070aed04d523ce286f4e 100644 --- a/demo/model_zoo/resnet/load_feature.py +++ b/demo/model_zoo/resnet/load_feature.py @@ -17,9 +17,11 @@ import sys import cPickle import logging -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + def load_feature_c(file): """ Load feature extracted by C++ interface. @@ -30,14 +32,15 @@ def load_feature_c(file): f = open(file, 'r') for line in f: sample = [] - for slot in line.strip().split(";"): - fea = [float(val) for val in slot.strip().split()] + for slot in line.strip().split(";"): + fea = [float(val) for val in slot.strip().split()] if fea: sample.append(fea) features.append(sample) f.close() return features + def load_feature_py(feature_dir): """ Load feature extracted by python interface. @@ -54,6 +57,7 @@ def load_feature_py(feature_dir): logging.info('Load feature file %s', file_name) return features + if __name__ == '__main__': - print load_feature_py(sys.argv[1]) + print load_feature_py(sys.argv[1]) #print load_feature_c(sys.argv[1]) diff --git a/demo/model_zoo/resnet/resnet.py b/demo/model_zoo/resnet/resnet.py index 483e308ac804e13ca249ef4e47e9e9b00770ce1b..015b74cd484596039b9fcf010576ca340d044db7 100644 --- a/demo/model_zoo/resnet/resnet.py +++ b/demo/model_zoo/resnet/resnet.py @@ -13,7 +13,6 @@ # limitations under the License. from paddle.trainer_config_helpers import * - """ paper: https://arxiv.org/abs/1512.03385 """ @@ -28,15 +27,19 @@ if not is_predict and data_provider: # mean.meta size : 3 x 224 x 224. # If you use three mean value, set like: # "mean_value:103.939,116.779,123.68;" - args={ + args = { 'mean_meta': "model/mean_meta_224/mean.meta", - 'image_size': 224, 'crop_size': 224, - 'color': True,'swap_channel:': [2, 1, 0]} - define_py_data_sources2(train_list, - 'example/test.list', - module="example.image_list_provider", - obj="processData", - args=args) + 'image_size': 224, + 'crop_size': 224, + 'color': True, + 'swap_channel:': [2, 1, 0] + } + define_py_data_sources2( + train_list, + 'example/test.list', + module="example.image_list_provider", + obj="processData", + args=args) batch_size = 1 learning_rate = 0.1 / batch_size @@ -54,12 +57,16 @@ Settings( learning_method='momentum', learning_rate_decay_a=0.5, learning_rate_decay_b=1200000 * 10, - learning_rate_schedule="discexp", -) + learning_rate_schedule="discexp", ) -def conv_bn_layer(name, input, filter_size, num_filters, - stride, padding, channels=None, +def conv_bn_layer(name, + input, + filter_size, + num_filters, + stride, + padding, + channels=None, active_type=ReluActivation()): """ A wrapper for conv layer with batch normalization layers. @@ -67,19 +74,18 @@ def conv_bn_layer(name, input, filter_size, num_filters, conv layer has no activation. """ - tmp = img_conv_layer(name=name + "_conv", - input=input, - filter_size=filter_size, - num_channels=channels, - num_filters=num_filters, - stride=stride, - padding=padding, - act=LinearActivation(), - bias_attr=False) - return batch_norm_layer(name=name + "_bn", - input=tmp, - act=active_type, - use_global_stats=is_test) + tmp = img_conv_layer( + name=name + "_conv", + input=input, + filter_size=filter_size, + num_channels=channels, + num_filters=num_filters, + stride=stride, + padding=padding, + act=LinearActivation(), + bias_attr=False) + return batch_norm_layer( + name=name + "_bn", input=tmp, act=active_type, use_global_stats=is_test) def bottleneck_block(name, input, num_filters1, num_filters2): @@ -88,29 +94,31 @@ def bottleneck_block(name, input, num_filters1, num_filters2): Last conv_bn_layer has no activation. Addto layer has activation of relu. """ - last_name = conv_bn_layer(name=name + '_branch2a', - input=input, - filter_size=1, - num_filters=num_filters1, - stride=1, - padding=0) - last_name = conv_bn_layer(name=name + '_branch2b', - input=last_name, - filter_size=3, - num_filters=num_filters1, - stride=1, - padding=1) - last_name = conv_bn_layer(name=name + '_branch2c', - input=last_name, - filter_size=1, - num_filters=num_filters2, - stride=1, - padding=0, - active_type=LinearActivation()) - - return addto_layer(name=name + "_addto", - input=[input, last_name], - act=ReluActivation()) + last_name = conv_bn_layer( + name=name + '_branch2a', + input=input, + filter_size=1, + num_filters=num_filters1, + stride=1, + padding=0) + last_name = conv_bn_layer( + name=name + '_branch2b', + input=last_name, + filter_size=3, + num_filters=num_filters1, + stride=1, + padding=1) + last_name = conv_bn_layer( + name=name + '_branch2c', + input=last_name, + filter_size=1, + num_filters=num_filters2, + stride=1, + padding=0, + active_type=LinearActivation()) + + return addto_layer( + name=name + "_addto", input=[input, last_name], act=ReluActivation()) def mid_projection(name, input, num_filters1, num_filters2, stride=2): @@ -123,38 +131,41 @@ def mid_projection(name, input, num_filters1, num_filters2, stride=2): branch2x: bottleneck building block, shortcuts are identity. """ # stride = 2 - branch1 = conv_bn_layer(name=name + '_branch1', - input=input, - filter_size=1, - num_filters=num_filters2, - stride=stride, - padding=0, - active_type=LinearActivation()) - - last_name = conv_bn_layer(name=name + '_branch2a', - input=input, - filter_size=1, - num_filters=num_filters1, - stride=stride, - padding=0) - last_name = conv_bn_layer(name=name + '_branch2b', - input=last_name, - filter_size=3, - num_filters=num_filters1, - stride=1, - padding=1) - - last_name = conv_bn_layer(name=name + '_branch2c', - input=last_name, - filter_size=1, - num_filters=num_filters2, - stride=1, - padding=0, - active_type=LinearActivation()) - - return addto_layer(name=name + "_addto", - input=[branch1, last_name], - act=ReluActivation()) + branch1 = conv_bn_layer( + name=name + '_branch1', + input=input, + filter_size=1, + num_filters=num_filters2, + stride=stride, + padding=0, + active_type=LinearActivation()) + + last_name = conv_bn_layer( + name=name + '_branch2a', + input=input, + filter_size=1, + num_filters=num_filters1, + stride=stride, + padding=0) + last_name = conv_bn_layer( + name=name + '_branch2b', + input=last_name, + filter_size=3, + num_filters=num_filters1, + stride=1, + padding=1) + + last_name = conv_bn_layer( + name=name + '_branch2c', + input=last_name, + filter_size=1, + num_filters=num_filters2, + stride=1, + padding=0, + active_type=LinearActivation()) + + return addto_layer( + name=name + "_addto", input=[branch1, last_name], act=ReluActivation()) def deep_res_net(res2_num=3, res3_num=4, res4_num=6, res5_num=3): @@ -168,67 +179,67 @@ def deep_res_net(res2_num=3, res3_num=4, res4_num=6, res5_num=3): # For ImageNet # conv1: 112x112 img = data_layer(name='input', size=224 * 224 * 3) - tmp = conv_bn_layer("conv1", img, - filter_size=7, - channels=3, - num_filters=64, - stride=2, - padding=3) + tmp = conv_bn_layer( + "conv1", + img, + filter_size=7, + channels=3, + num_filters=64, + stride=2, + padding=3) tmp = img_pool_layer(name="pool1", input=tmp, pool_size=3, stride=2) # conv2_x: 56x56 - tmp = mid_projection(name="res2_1", - input=tmp, - num_filters1=64, - num_filters2=256, - stride=1) + tmp = mid_projection( + name="res2_1", input=tmp, num_filters1=64, num_filters2=256, stride=1) for i in xrange(2, res2_num + 1, 1): - tmp = bottleneck_block(name="res2_" + str(i), - input=tmp, - num_filters1=64, - num_filters2=256) + tmp = bottleneck_block( + name="res2_" + str(i), input=tmp, num_filters1=64, num_filters2=256) # conv3_x: 28x28 - tmp = mid_projection(name="res3_1", - input=tmp, - num_filters1=128, - num_filters2=512) + tmp = mid_projection( + name="res3_1", input=tmp, num_filters1=128, num_filters2=512) for i in xrange(2, res3_num + 1, 1): - tmp = bottleneck_block(name="res3_" + str(i), - input=tmp, num_filters1=128, - num_filters2=512) + tmp = bottleneck_block( + name="res3_" + str(i), + input=tmp, + num_filters1=128, + num_filters2=512) # conv4_x: 14x14 - tmp = mid_projection(name="res4_1", input=tmp, - num_filters1=256, num_filters2=1024) + tmp = mid_projection( + name="res4_1", input=tmp, num_filters1=256, num_filters2=1024) for i in xrange(2, res4_num + 1, 1): - tmp = bottleneck_block(name="res4_" + str(i), - input=tmp, - num_filters1=256, - num_filters2=1024) + tmp = bottleneck_block( + name="res4_" + str(i), + input=tmp, + num_filters1=256, + num_filters2=1024) # conv5_x: 7x7 - tmp = mid_projection(name="res5_1", input=tmp, - num_filters1=512, num_filters2=2048) + tmp = mid_projection( + name="res5_1", input=tmp, num_filters1=512, num_filters2=2048) for i in xrange(2, res5_num + 1, 1): - tmp = bottleneck_block(name="res5_" + str(i), - input=tmp, num_filters1=512, - num_filters2=2048) - - tmp = img_pool_layer(name='avgpool', - input=tmp, - pool_size=7, - stride=1, - pool_type=AvgPooling()) - - output = fc_layer(name='output', - input=tmp, - size=1000, - act=SoftmaxActivation()) + tmp = bottleneck_block( + name="res5_" + str(i), + input=tmp, + num_filters1=512, + num_filters2=2048) + + tmp = img_pool_layer( + name='avgpool', + input=tmp, + pool_size=7, + stride=1, + pool_type=AvgPooling()) + + output = fc_layer( + name='output', input=tmp, size=1000, act=SoftmaxActivation()) if not is_predict: - classification_cost(input=output, label=data_layer(name='label', - size=1)) + classification_cost( + input=output, label=data_layer( + name='label', size=1)) def res_net_50(): diff --git a/demo/quick_start/api_train.py b/demo/quick_start/api_train.py index 5ae19b8d26534a9521a6da7630796edce36780e7..66cbb856484d231613a0026be129a7bc3a7cfdf5 100644 --- a/demo/quick_start/api_train.py +++ b/demo/quick_start/api_train.py @@ -22,27 +22,32 @@ from py_paddle import DataProviderConverter from paddle.trainer.PyDataProvider2 \ import integer_value, integer_value_sequence, sparse_binary_vector + def parse_arguments(): parser = argparse.ArgumentParser() - parser.add_argument("--train_data", - type=str, required=False, help="train data file") + parser.add_argument( + "--train_data", type=str, required=False, help="train data file") parser.add_argument("--test_data", type=str, help="test data file") - parser.add_argument("--config", - type=str, required=True, help="config file name") + parser.add_argument( + "--config", type=str, required=True, help="config file name") parser.add_argument("--dict_file", required=True, help="dictionary file") - parser.add_argument("--seq", - default=1, type=int, - help="whether use sequence training") - parser.add_argument("--use_gpu", default=0, type=int, - help="whether use GPU for training") - parser.add_argument("--trainer_count", default=1, type=int, - help="Number of threads for training") - parser.add_argument("--num_passes", default=5, type=int, - help="Number of training passes") + parser.add_argument( + "--seq", default=1, type=int, help="whether use sequence training") + parser.add_argument( + "--use_gpu", default=0, type=int, help="whether use GPU for training") + parser.add_argument( + "--trainer_count", + default=1, + type=int, + help="Number of threads for training") + parser.add_argument( + "--num_passes", default=5, type=int, help="Number of training passes") return parser.parse_args() + UNK_IDX = 0 + def load_data(file_name, word_dict): with open(file_name, 'r') as f: for line in f: @@ -51,6 +56,7 @@ def load_data(file_name, word_dict): word_slot = [word_dict.get(w, UNK_IDX) for w in words] yield word_slot, int(label) + def load_dict(dict_file): word_dict = dict() with open(dict_file, 'r') as f: @@ -59,6 +65,7 @@ def load_dict(dict_file): word_dict[w] = i return word_dict + def main(): options = parse_arguments() api.initPaddle("--use_gpu=%s" % options.use_gpu, @@ -86,9 +93,9 @@ def main(): # create a data converter which converts data to PaddlePaddle # internal format input_types = [ - integer_value_sequence(len(word_dict)) if options.seq - else sparse_binary_vector(len(word_dict)), - integer_value(2)] + integer_value_sequence(len(word_dict)) if options.seq else + sparse_binary_vector(len(word_dict)), integer_value(2) + ] converter = DataProviderConverter(input_types) batch_size = trainer_config.opt_config.batch_size @@ -102,7 +109,7 @@ def main(): trainer.trainOneDataBatch(size, converter(batch)) trainer.finishTrainPass() if test_dataset: - trainer.startTestPeriod(); + trainer.startTestPeriod() for pos in xrange(0, len(test_dataset), batch_size): batch = itertools.islice(test_dataset, pos, pos + batch_size) size = min(batch_size, len(test_dataset) - pos) @@ -110,5 +117,6 @@ def main(): trainer.finishTestPeriod() trainer.finishTrain() + if __name__ == '__main__': main() diff --git a/demo/quick_start/dataprovider_bow.py b/demo/quick_start/dataprovider_bow.py index f8cde189cf87d73aec05da4b34e064cddecff56b..a5156a2d40cc04c02e50d676045ae6da8937ba01 100644 --- a/demo/quick_start/dataprovider_bow.py +++ b/demo/quick_start/dataprovider_bow.py @@ -17,6 +17,7 @@ from paddle.trainer.PyDataProvider2 import * # id of the word not in dictionary UNK_IDX = 0 + # initializer is called by the framework during initialization. # It allows the user to describe the data types and setup the # necessary data structure for later use. @@ -38,7 +39,9 @@ def initializer(settings, dictionary, **kwargs): # The second input is an integer. It represents the category id of the # sample. 2 means there are two labels in the dataset. # (1 for positive and 0 for negative) - integer_value(2)] + integer_value(2) + ] + # Delaring a data provider. It has an initializer 'data_initialzer'. # It will cache the generated data of the first pass in memory, so that @@ -69,9 +72,8 @@ def process(settings, file_name): def predict_initializer(settings, dictionary, **kwargs): settings.word_dict = dictionary - settings.input_types = [ - sparse_binary_vector(len(dictionary)) - ] + settings.input_types = [sparse_binary_vector(len(dictionary))] + # Declaring a data provider for prediction. The difference with process # is that label is not generated. diff --git a/demo/quick_start/dataprovider_emb.py b/demo/quick_start/dataprovider_emb.py index f5632d5f3f8bd8bb83b12198e7450b239eb1f7f6..286f3f5c82081f1a6e02a26023969790792a78a3 100755 --- a/demo/quick_start/dataprovider_emb.py +++ b/demo/quick_start/dataprovider_emb.py @@ -24,7 +24,8 @@ def initializer(settings, dictionary, **kwargs): # The value of the integers range from 0 to len(dictrionary)-1 integer_value_sequence(len(dictionary)), # Define the second input for label id - integer_value(2)] + integer_value(2) + ] @provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) @@ -40,7 +41,8 @@ def process(settings, file_name): def predict_initializer(settings, dictionary, **kwargs): settings.word_dict = dictionary settings.input_types = [ - integer_value(len(dictionary), seq_type=SequenceType.SEQUENCE) + integer_value( + len(dictionary), seq_type=SequenceType.SEQUENCE) ] diff --git a/demo/quick_start/preprocess.py b/demo/quick_start/preprocess.py index 69fdbe44b5245bc2855847a1507e6eaed517eb96..d87fad632a7429f7d9682badabe4c72ca127354f 100755 --- a/demo/quick_start/preprocess.py +++ b/demo/quick_start/preprocess.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ 1. (remove HTML before or not)tokensizing 2. pos sample : rating score 5; neg sample: rating score 1-2. @@ -35,7 +34,8 @@ import multiprocessing batch_size = 5000 word_count = {} -num_tokenize = max(1, multiprocessing.cpu_count() - 2) # parse + tokenize + save +num_tokenize = max(1, + multiprocessing.cpu_count() - 2) # parse + tokenize + save max_queue_size = 8 parse_queue = Queue(maxsize=max_queue_size + num_tokenize) tokenize_queue = Queue(maxsize=max_queue_size + num_tokenize) diff --git a/demo/quick_start/train.sh b/demo/quick_start/train.sh index 49806292a4ec5bd4194ccb6f6a638b6b2b4f37ed..b3c471608c3248bfc714d5e44dd927f25dd23ea0 100755 --- a/demo/quick_start/train.sh +++ b/demo/quick_start/train.sh @@ -20,6 +20,7 @@ cfg=trainer_config.lr.py #cfg=trainer_config.lstm.py #cfg=trainer_config.bidi-lstm.py #cfg=trainer_config.db-lstm.py +#cfg=trainer_config.resnet-lstm.py paddle train \ --config=$cfg \ --save_dir=./output \ diff --git a/demo/quick_start/trainer_config.bidi-lstm.py b/demo/quick_start/trainer_config.bidi-lstm.py index 3be3d373422714c6b40e530cf112f9106b85d20b..51deaf31f94681b6b61f98f798cef14a65ec92cb 100644 --- a/demo/quick_start/trainer_config.bidi-lstm.py +++ b/demo/quick_start/trainer_config.bidi-lstm.py @@ -27,11 +27,12 @@ is_predict = get_config_arg('is_predict', bool, False) trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,19 +40,17 @@ settings( learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) -bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) +bias_attr = ParamAttr(initial_std=0., l2_rate=0.) data = data_layer(name="word", size=len(word_dict)) emb = embedding_layer(input=data, size=128) bi_lstm = bidirectional_lstm(input=emb, size=128) dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) -output = fc_layer(input=dropout, size=2, - bias_attr=bias_attr, - act=SoftmaxActivation()) +output = fc_layer( + input=dropout, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) if is_predict: maxid = maxid_layer(output) diff --git a/demo/quick_start/trainer_config.cnn.py b/demo/quick_start/trainer_config.cnn.py index 253ec0aee26cf42226d79726a75aad6c61c77565..388efa75f903e0c7c803c99cd50d73a004133a67 100644 --- a/demo/quick_start/trainer_config.cnn.py +++ b/demo/quick_start/trainer_config.cnn.py @@ -27,11 +27,12 @@ is_predict = get_config_arg('is_predict', bool, False) trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,8 +40,7 @@ settings( learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) data = data_layer(name="word", size=len(word_dict)) embedding = embedding_layer(input=data, size=128) diff --git a/demo/quick_start/trainer_config.db-lstm.py b/demo/quick_start/trainer_config.db-lstm.py index b35bdf5a61b4731cadb5eb992796c5e885efbd7e..02bc898d881efbd3bfaed95d45cd9e70ed046746 100644 --- a/demo/quick_start/trainer_config.db-lstm.py +++ b/demo/quick_start/trainer_config.db-lstm.py @@ -27,11 +27,12 @@ is_predict = get_config_arg('is_predict', bool, False) trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,10 +40,9 @@ settings( learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) -bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) +bias_attr = ParamAttr(initial_std=0., l2_rate=0.) data = data_layer(name="word", size=len(word_dict)) emb = embedding_layer(input=data, size=128) @@ -52,17 +52,18 @@ lstm_0 = lstmemory(input=hidden_0, layer_attr=ExtraAttr(drop_rate=0.1)) input_layers = [hidden_0, lstm_0] -for i in range(1,8): +for i in range(1, 8): fc = fc_layer(input=input_layers, size=128) - lstm = lstmemory(input=fc, layer_attr=ExtraAttr(drop_rate=0.1), - reverse=(i % 2) == 1,) + lstm = lstmemory( + input=fc, + layer_attr=ExtraAttr(drop_rate=0.1), + reverse=(i % 2) == 1, ) input_layers = [fc, lstm] lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) -output = fc_layer(input=lstm_last, size=2, - bias_attr=bias_attr, - act=SoftmaxActivation()) +output = fc_layer( + input=lstm_last, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) if is_predict: maxid = maxid_layer(output) diff --git a/demo/quick_start/trainer_config.emb.py b/demo/quick_start/trainer_config.emb.py index 34dd7b96f2f142159472b98a09fd0092fac15e43..8fd18a7aac704e62b137845edb46cce5bc373285 100644 --- a/demo/quick_start/trainer_config.emb.py +++ b/demo/quick_start/trainer_config.emb.py @@ -27,18 +27,16 @@ is_predict = get_config_arg('is_predict', bool, False) trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer() -) + batch_size=batch_size, learning_rate=2e-3, learning_method=AdamOptimizer()) data = data_layer(name="word", size=len(word_dict)) embedding = embedding_layer(input=data, size=128) diff --git a/demo/quick_start/trainer_config.lr.py b/demo/quick_start/trainer_config.lr.py index c6059947f30b32975d72155150de095ade01aa9d..b9c9441baac28a8a8f6078065b75664819d6cd04 100644 --- a/demo/quick_start/trainer_config.lr.py +++ b/demo/quick_start/trainer_config.lr.py @@ -32,11 +32,12 @@ process = 'process' if not is_predict else 'process_predict' # We need to use different process for training and prediction. # For training, the input data includes both word IDs and labels. # For prediction, the input data only includs word Ids. -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_bow", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_bow", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -44,8 +45,7 @@ settings( learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) # Define the data for text features. The size of the data layer is the number # of words in the dictionary. diff --git a/demo/quick_start/trainer_config.lstm.py b/demo/quick_start/trainer_config.lstm.py index b412a9cbd914dc7abd70b93bbe250759552ee071..8821e02d9bd4a0d06b8afa99df8e0fac3e2fcefe 100644 --- a/demo/quick_start/trainer_config.lstm.py +++ b/demo/quick_start/trainer_config.lstm.py @@ -27,11 +27,12 @@ is_predict = get_config_arg('is_predict', bool, False) trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,17 +40,14 @@ settings( learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) - + gradient_clipping_threshold=25) data = data_layer(name="word", size=len(word_dict)) emb = embedding_layer(input=data, size=128) -lstm = simple_lstm(input=emb, size=128, - lstm_cell_attr=ExtraAttr(drop_rate=0.25)) +lstm = simple_lstm( + input=emb, size=128, lstm_cell_attr=ExtraAttr(drop_rate=0.25)) lstm_max = pooling_layer(input=lstm, pooling_type=MaxPooling()) -output = fc_layer(input=lstm_max, size=2, - act=SoftmaxActivation()) +output = fc_layer(input=lstm_max, size=2, act=SoftmaxActivation()) if is_predict: maxid = maxid_layer(output) outputs([maxid, output]) diff --git a/demo/quick_start/trainer_config.resnet-lstm.py b/demo/quick_start/trainer_config.resnet-lstm.py new file mode 100644 index 0000000000000000000000000000000000000000..91e1581c386eb880d481b7352c4d21f3a5ef5c9a --- /dev/null +++ b/demo/quick_start/trainer_config.resnet-lstm.py @@ -0,0 +1,94 @@ +# edit-mode: -*- python -*- + +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This configuration is a demonstration of how to implement the stacked LSTM +with residual connections, i.e. an LSTM layer takes the sum of the hidden states +and inputs of the previous LSTM layer instead of only the hidden states. +This architecture is from: +Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc V. Le, Mohammad Norouzi, +Wolfgang Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey, +Jeff Klingner, Apurva Shah, Melvin Johnson, Xiaobing Liu, Lukasz Kaiser, +Stephan Gouws, Yoshikiyo Kato, Taku Kudo, Hideto Kazawa, Keith Stevens, +George Kurian, Nishant Patil, Wei Wang, Cliff Young, Jason Smith, Jason Riesa, +Alex Rudnick, Oriol Vinyals, Greg Corrado, Macduff Hughes, Jeffrey Dean. 2016. +Google's Neural Machine Translation System: Bridging the Gap between Human and +Machine Translation. In arXiv https://arxiv.org/pdf/1609.08144v2.pdf +Different from the architecture described in the paper, we use a stack single +direction LSTM layers as the first layer instead of bi-directional LSTM. Also, +since this is a demo code, to reduce computation time, we stacked 4 layers +instead of 8 layers. +""" + +from paddle.trainer_config_helpers import * + +dict_file = "./data/dict.txt" +word_dict = dict() +with open(dict_file, 'r') as f: + for i, line in enumerate(f): + w = line.strip().split()[0] + word_dict[w] = i + +is_predict = get_config_arg('is_predict', bool, False) +trn = 'data/train.list' if not is_predict else None +tst = 'data/test.list' if not is_predict else 'data/pred.list' +process = 'process' if not is_predict else 'process_predict' +define_py_data_sources2(train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) + +batch_size = 128 if not is_predict else 1 +settings( + batch_size=batch_size, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25 +) + +bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) + +data = data_layer(name="word", size=len(word_dict)) +emb = embedding_layer(input=data, size=128) +lstm = simple_lstm(input=emb, size=128, lstm_cell_attr=ExtraAttr(drop_rate=0.1)) + +previous_input, previous_hidden_state = emb, lstm + +for i in range(3): + # The input to the current layer is the sum of the hidden state + # and input of the previous layer. + current_input = addto_layer(input=[previous_input, previous_hidden_state]) + hidden_state = simple_lstm(input=current_input, size=128, + lstm_cell_attr=ExtraAttr(drop_rate=0.1)) + previous_input, previous_hidden_state = current_input, hidden_state + +lstm = previous_hidden_state + +lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) +output = fc_layer(input=lstm_last, size=2, + bias_attr=bias_attr, + act=SoftmaxActivation()) + + +if is_predict: + maxid = maxid_layer(output) + outputs([maxid, output]) +else: + label = data_layer(name="label", size=2) + cls = classification_cost(input=output, label=label) + outputs(cls) diff --git a/demo/recommendation/common_utils.py b/demo/recommendation/common_utils.py index a5f00b3ef9ca00b42b8e31ddd6cfeca3580152b0..613e36b496e47edbc0eabd8f15a0abdcb50f6424 100755 --- a/demo/recommendation/common_utils.py +++ b/demo/recommendation/common_utils.py @@ -21,8 +21,9 @@ def meta_to_header(meta, name): yield integer_value(each_meta['max']) elif each_meta['type'] == 'embedding': is_seq = each_meta['seq'] == 'sequence' - yield integer_value(len(each_meta['dict']), - seq_type=SequenceType.SEQUENCE if is_seq - else SequenceType.NO_SEQUENCE) + yield integer_value( + len(each_meta['dict']), + seq_type=SequenceType.SEQUENCE + if is_seq else SequenceType.NO_SEQUENCE) elif each_meta['type'] == 'one_hot_dense': yield dense_vector(len(each_meta['dict'])) diff --git a/demo/recommendation/data/config.json b/demo/recommendation/data/config.json index 71a9dd7be6bd10e177dfb443a94b719c3816d833..f26e74ce47bb7843a571e6033f051c046b31f054 100644 --- a/demo/recommendation/data/config.json +++ b/demo/recommendation/data/config.json @@ -14,4 +14,3 @@ "fields": ["id", "title", "genres"] } } - diff --git a/demo/recommendation/data/config_generator.py b/demo/recommendation/data/config_generator.py index 29f38082693ad890ac4dfa302399663af8dbd27b..fa605458300f81da6772d88cfbad413e4dcf97fe 100644 --- a/demo/recommendation/data/config_generator.py +++ b/demo/recommendation/data/config_generator.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ config_generator.py @@ -29,10 +28,7 @@ import json import docopt import copy -DEFAULT_FILE = { - "type": "split", - "delimiter": "," -} +DEFAULT_FILE = {"type": "split", "delimiter": ","} DEFAULT_FIELD = { "id": { @@ -107,19 +103,16 @@ def main(filename, fmt): field = copy.deepcopy(DEFAULT_FIELD[field_key]) field['pos'] = pos fields.append(field) - obj[k] = { - "file": file_dict, - "fields": fields - } - meta = { - "meta": obj - } + obj[k] = {"file": file_dict, "fields": fields} + meta = {"meta": obj} # print meta if fmt == 'json': + def formatter(x): import json return json.dumps(x, indent=2) elif fmt == 'yaml': + def formatter(x): import yaml return yaml.safe_dump(x, default_flow_style=False) diff --git a/demo/recommendation/data/meta_generator.py b/demo/recommendation/data/meta_generator.py index 8d1a33d02aea112e51f1d43bedc06fdcee1186f5..593c863670d5eb5d684adf643ff745f3914b656b 100644 --- a/demo/recommendation/data/meta_generator.py +++ b/demo/recommendation/data/meta_generator.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Preprocess Movielens dataset, to get movie/user object. @@ -66,8 +65,8 @@ class SortedIDGenerator(object): self.__key_set__.add(key) def finish_scan(self, compare=None, key=None, reverse=False): - self.__key_set__ = sorted(list(self.__key_set__), cmp=compare, - key=key, reverse=reverse) + self.__key_set__ = sorted( + list(self.__key_set__), cmp=compare, key=key, reverse=reverse) self.dict = dict() for idx, each_key in enumerate(self.__key_set__): self.dict[each_key] = idx @@ -207,11 +206,10 @@ class EmbeddingFieldParser(object): self.dict = EmbeddingFieldParser.CharBasedEmbeddingDict( self.seq_type == EmbeddingFieldParser.SEQUENCE) elif config['dict']['type'] == 'split': - self.dict = SplitEmbeddingDict( - config['dict'].get('delimiter', ',')) + self.dict = SplitEmbeddingDict(config['dict'].get('delimiter', ',')) elif config['dict']['type'] == 'whole_content': - self.dict = EmbeddingFieldParser.WholeContentDict( - config['dict']['sort']) + self.dict = EmbeddingFieldParser.WholeContentDict(config['dict'][ + 'sort']) else: print config assert False @@ -333,8 +331,8 @@ class ContentExtractorFactory(object): return PositionContentExtractor(config['pos']) else: extra_args = config['regex'] - return RegexPositionContentExtractor(pos=config['pos'], - **extra_args) + return RegexPositionContentExtractor( + pos=config['pos'], **extra_args) class MetaFile(object): @@ -364,9 +362,10 @@ class MetaFile(object): metas = map(lambda x: x.meta_field(), field_parsers) # print metas - key_index = filter(lambda x: x is not None, map( - lambda (idx, meta): idx if 'is_key' in meta and meta['is_key'] - else None, enumerate(metas)))[0] + key_index = filter( + lambda x: x is not None, + map(lambda (idx, meta): idx if 'is_key' in meta and meta['is_key'] else None, + enumerate(metas)))[0] key_map = [] for i in range(min(key_index, len(metas))): @@ -374,12 +373,7 @@ class MetaFile(object): for i in range(key_index + 1, len(metas)): key_map.append(i) - obj = { - '__meta__': { - 'raw_meta': metas, - 'feature_map': key_map - } - } + obj = {'__meta__': {'raw_meta': metas, 'feature_map': key_map}} for each_block in reader.read(): idx = field_parsers[key_index].parse(each_block) diff --git a/demo/recommendation/data/split.py b/demo/recommendation/data/split.py index ff1f7fab7befdb5bdfa39fd0f1753e6804e82d8f..8dd0cbd32af6074439e98dac024c5fed76cd52b2 100644 --- a/demo/recommendation/data/split.py +++ b/demo/recommendation/data/split.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Separate movielens 1m dataset to train/test file. diff --git a/demo/recommendation/dataprovider.py b/demo/recommendation/dataprovider.py index 454467f40b44bb526d143934c4a7350d41e54c0e..ff3932be03f1e4a1fc1d0bdb189ab7fe1fbbeca0 100755 --- a/demo/recommendation/dataprovider.py +++ b/demo/recommendation/dataprovider.py @@ -15,6 +15,7 @@ from paddle.trainer.PyDataProvider2 import * import common_utils # parse + def hook(settings, meta, **kwargs): """ Init hook is invoked before process data. It will set obj.slots and store @@ -41,6 +42,7 @@ def hook(settings, meta, **kwargs): settings.input_types = headers settings.meta = meta + @provider(init_hook=hook, cache=CacheType.CACHE_PASS_IN_MEM) def process(settings, filename): with open(filename, 'r') as f: diff --git a/demo/recommendation/prediction.py b/demo/recommendation/prediction.py index f8044a3195ec25bc2fa7c9079e4977f971059352..e2a202cfd1a476046d7e1d1896b87d72c4906ff2 100755 --- a/demo/recommendation/prediction.py +++ b/demo/recommendation/prediction.py @@ -28,7 +28,8 @@ if __name__ == '__main__': model_path = sys.argv[1] swig_paddle.initPaddle('--use_gpu=0') conf = parse_config("trainer_config.py", "is_predict=1") - network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) network.loadParameters(model_path) with open('./data/meta.bin', 'rb') as f: @@ -39,11 +40,12 @@ if __name__ == '__main__': while True: movie_id = int(raw_input("Input movie_id: ")) user_id = int(raw_input("Input user_id: ")) - movie_meta = meta['movie'][movie_id] # Query Data From Meta. + movie_meta = meta['movie'][movie_id] # Query Data From Meta. user_meta = meta['user'][user_id] data = [movie_id - 1] data.extend(movie_meta) data.append(user_id - 1) data.extend(user_meta) - print "Prediction Score is %.2f" % ((network.forwardTest( - cvt.convert([data]))[0]['value'][0][0] + 5) / 2) + print "Prediction Score is %.2f" % ( + (network.forwardTest(cvt.convert([data]))[0]['value'][0][0] + 5) + / 2) diff --git a/demo/recommendation/trainer_config.py b/demo/recommendation/trainer_config.py index 624c22ec969dc98808863ad53573b9633f1791ac..cec340b0b65a841029a1c0538d9881bb38f026ff 100755 --- a/demo/recommendation/trainer_config.py +++ b/demo/recommendation/trainer_config.py @@ -27,8 +27,8 @@ with open(META_FILE, 'rb') as f: # load meta file meta = pickle.load(f) -settings(batch_size=1600, learning_rate=1e-3, - learning_method=RMSPropOptimizer()) +settings( + batch_size=1600, learning_rate=1e-3, learning_method=RMSPropOptimizer()) def construct_feature(name): @@ -59,11 +59,10 @@ def construct_feature(name): slot_name = each_meta.get('name', '%s_id' % name) if type_name == 'id': slot_dim = each_meta['max'] - embedding = embedding_layer(input=data_layer(slot_name, - size=slot_dim), - size=256) - fusion.append(fc_layer(input=embedding, - size=256)) + embedding = embedding_layer( + input=data_layer( + slot_name, size=slot_dim), size=256) + fusion.append(fc_layer(input=embedding, size=256)) elif type_name == 'embedding': is_seq = each_meta['seq'] == 'sequence' slot_dim = len(each_meta['dict']) @@ -71,17 +70,14 @@ def construct_feature(name): embedding = embedding_layer(input=din, size=256) if is_seq: fusion.append( - text_conv_pool(input=embedding, context_len=5, - hidden_size=256)) + text_conv_pool( + input=embedding, context_len=5, hidden_size=256)) else: - fusion.append(fc_layer(input=embedding, - size=256)) + fusion.append(fc_layer(input=embedding, size=256)) elif type_name == 'one_hot_dense': slot_dim = len(each_meta['dict']) - hidden = fc_layer(input=data_layer(slot_name, slot_dim), - size=256) - fusion.append(fc_layer(input=hidden, - size=256)) + hidden = fc_layer(input=data_layer(slot_name, slot_dim), size=256) + fusion.append(fc_layer(input=hidden, size=256)) return fc_layer(name="%s_fusion" % name, input=fusion, size=256) @@ -90,10 +86,16 @@ movie_feature = construct_feature("movie") user_feature = construct_feature("user") similarity = cos_sim(a=movie_feature, b=user_feature) if not is_predict: - outputs(regression_cost(input=similarity, - label=data_layer('rating', size=1))) - - define_py_data_sources2('data/train.list', 'data/test.list', module='dataprovider', - obj='process', args={'meta': meta}) + outputs( + regression_cost( + input=similarity, label=data_layer( + 'rating', size=1))) + + define_py_data_sources2( + 'data/train.list', + 'data/test.list', + module='dataprovider', + obj='process', + args={'meta': meta}) else: outputs(similarity) diff --git a/demo/semantic_role_labeling/dataprovider.py b/demo/semantic_role_labeling/dataprovider.py index 2ef25c42c1794c410fe85fd497a6ed9d2295dca9..5c003584a52d459f13b7942ebe3a7147ac58a42f 100644 --- a/demo/semantic_role_labeling/dataprovider.py +++ b/demo/semantic_role_labeling/dataprovider.py @@ -26,9 +26,9 @@ def hook(settings, word_dict, label_dict, **kwargs): integer_value_sequence(len(word_dict)), integer_value_sequence(len(word_dict)), integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(2), - integer_value_sequence(len(label_dict))] + integer_value_sequence(len(word_dict)), integer_value_sequence(2), + integer_value_sequence(len(label_dict)) + ] @provider(init_hook=hook) diff --git a/demo/semantic_role_labeling/db_lstm.py b/demo/semantic_role_labeling/db_lstm.py index 364460afbe31caf42cd4f0836eba75e444b3f5b8..e3f6edad6972112ed04e173a9b714e3fec13d402 100644 --- a/demo/semantic_role_labeling/db_lstm.py +++ b/demo/semantic_role_labeling/db_lstm.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import math import os import sys @@ -42,7 +41,7 @@ if not is_predict: label_dict[w] = i if is_test: - train_list_file = None + train_list_file = None #define data provider define_py_data_sources2( diff --git a/demo/semantic_role_labeling/predict.py b/demo/semantic_role_labeling/predict.py index 9a27112828e449174e3da79dc7db9fed20bfed6f..f051d4175cf6fff43bd7f84b457ab9dd12405a15 100644 --- a/demo/semantic_role_labeling/predict.py +++ b/demo/semantic_role_labeling/predict.py @@ -41,22 +41,16 @@ class Prediction(): len_dict = len(self.dict) len_label = len(self.labels) - conf = parse_config( - train_conf, - 'dict_len=' + str(len_dict) + - ',label_len=' + str(len_label) + - ',is_predict=True') + conf = parse_config(train_conf, 'dict_len=' + str(len_dict) + + ',label_len=' + str(len_label) + ',is_predict=True') self.network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) self.network.loadParameters(model_dir) slots = [ - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(2) + integer_value_sequence(len_dict), integer_value_sequence(len_dict), + integer_value_sequence(len_dict), integer_value_sequence(len_dict), + integer_value_sequence(len_dict), integer_value_sequence(2) ] self.converter = DataProviderConverter(slots) @@ -110,8 +104,8 @@ class Prediction(): len_sen = len(sen.split()) line_labels = lab[index:index + len_sen] index += len_sen - fout.write(sen + '\t' + ' '.join([self.labels_reverse[ - i] for i in line_labels]) + '\n') + fout.write(sen + '\t' + ' '.join( + [self.labels_reverse[i] for i in line_labels]) + '\n') def option_parser(): diff --git a/demo/semantic_role_labeling/test.sh b/demo/semantic_role_labeling/test.sh index 804f722e5b8e9ee5b54c778c54f7833f5e6c4de0..844649e8c0f6867dc0766e4ec6f250c5a4a004d9 100644 --- a/demo/semantic_role_labeling/test.sh +++ b/demo/semantic_role_labeling/test.sh @@ -37,4 +37,3 @@ paddle train \ --use_gpu=false \ --config_args=is_test=1 \ 2>&1 | tee 'test.log' - diff --git a/demo/semantic_role_labeling/train.sh b/demo/semantic_role_labeling/train.sh index 94c7b6f31df3b5e5e059d6e1323ae0c0bec74753..c3a22b644be0ca08a2af73a57c09657014e49bfc 100644 --- a/demo/semantic_role_labeling/train.sh +++ b/demo/semantic_role_labeling/train.sh @@ -24,4 +24,3 @@ paddle train \ --show_parameter_stats_period=10 \ --test_all_data_in_one_period=1 \ 2>&1 | tee 'train.log' - diff --git a/demo/sentiment/data/get_imdb.sh b/demo/sentiment/data/get_imdb.sh index 41523927afe75428ef1151cef8184ede14eea9a7..28fa86232d89964b3f1680080239cf8a4ebefa9a 100755 --- a/demo/sentiment/data/get_imdb.sh +++ b/demo/sentiment/data/get_imdb.sh @@ -38,11 +38,11 @@ unzip master.zip mkdir -p imdb/train mkdir -p imdb/test -cp -r aclImdb/train/pos/ imdb/train/ -cp -r aclImdb/train/neg/ imdb/train/ +cp -r aclImdb/train/pos/ imdb/train/pos +cp -r aclImdb/train/neg/ imdb/train/neg -cp -r aclImdb/test/pos/ imdb/test/ -cp -r aclImdb/test/neg/ imdb/test/ +cp -r aclImdb/test/pos/ imdb/test/pos +cp -r aclImdb/test/neg/ imdb/test/neg #remove compressed package rm aclImdb_v1.tar.gz diff --git a/demo/sentiment/dataprovider.py b/demo/sentiment/dataprovider.py index 9a9fd81f030cb1d2a10a5000fd1d12810d12112b..53e3d1d20df92b8815347bd8937064871f326b3f 100755 --- a/demo/sentiment/dataprovider.py +++ b/demo/sentiment/dataprovider.py @@ -17,8 +17,8 @@ from paddle.trainer.PyDataProvider2 import * def hook(settings, dictionary, **kwargs): settings.word_dict = dictionary settings.input_types = [ - integer_value_sequence(len(settings.word_dict)), - integer_value(2)] + integer_value_sequence(len(settings.word_dict)), integer_value(2) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -29,6 +29,7 @@ def process(settings, file_name): label, comment = line.strip().split('\t\t') label = int(label) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if w in - settings.word_dict] + word_slot = [ + settings.word_dict[w] for w in words if w in settings.word_dict + ] yield word_slot, label diff --git a/demo/sentiment/predict.py b/demo/sentiment/predict.py index 7d0baeabbba68b2a160463364d05cd865bf0314f..bc0f6f31264294034ed38309f7fda370865b2845 100755 --- a/demo/sentiment/predict.py +++ b/demo/sentiment/predict.py @@ -18,14 +18,14 @@ from optparse import OptionParser from py_paddle import swig_paddle, DataProviderConverter from paddle.trainer.PyDataProvider2 import integer_value_sequence from paddle.trainer.config_parser import parse_config - """ Usage: run following command to show help message. python predict.py -h """ + class SentimentPrediction(): - def __init__(self, train_conf, dict_file, model_dir=None, label_file = None): + def __init__(self, train_conf, dict_file, model_dir=None, label_file=None): """ train_conf: trainer configure. dict_file: word dictionary file name. @@ -44,7 +44,8 @@ class SentimentPrediction(): self.load_label(label_file) conf = parse_config(train_conf, "is_predict=1") - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) self.network.loadParameters(self.model_dir) input_types = [integer_value_sequence(self.dict_dim)] self.converter = DataProviderConverter(input_types) @@ -61,7 +62,7 @@ class SentimentPrediction(): """ Load label. """ - self.label={} + self.label = {} for v in open(label_file, 'r'): self.label[int(v.split('\t')[1])] = v.split('\t')[0] @@ -72,7 +73,9 @@ class SentimentPrediction(): with open(data_file, 'r') as fdata: for line in fdata: words = line.strip().split() - word_slot = [self.word_dict[w] for w in words if w in self.word_dict] + word_slot = [ + self.word_dict[w] for w in words if w in self.word_dict + ] if not word_slot: print "all words are not in dictionary: %s", line continue @@ -89,25 +92,48 @@ class SentimentPrediction(): if self.label is None: print("%s: predicting label is %d" % (data_file, lab[0][0])) else: - print("%s: predicting label is %s" % (data_file, self.label[lab[0][0]])) + print("%s: predicting label is %s" % + (data_file, self.label[lab[0][0]])) + def option_parser(): usage = "python predict.py -n config -w model_dir -d dictionary -i input_file " parser = OptionParser(usage="usage: %s [options]" % usage) - parser.add_option("-n", "--tconf", action="store", - dest="train_conf", help="network config") - parser.add_option("-d", "--dict", action="store", - dest="dict_file",help="dictionary file") - parser.add_option("-b", "--label", action="store", - dest="label", default=None, - help="dictionary file") - parser.add_option("-i", "--data", action="store", - dest="data", help="data file to predict") - parser.add_option("-w", "--model", action="store", - dest="model_path", default=None, - help="model path") + parser.add_option( + "-n", + "--tconf", + action="store", + dest="train_conf", + help="network config") + parser.add_option( + "-d", + "--dict", + action="store", + dest="dict_file", + help="dictionary file") + parser.add_option( + "-b", + "--label", + action="store", + dest="label", + default=None, + help="dictionary file") + parser.add_option( + "-i", + "--data", + action="store", + dest="data", + help="data file to predict") + parser.add_option( + "-w", + "--model", + action="store", + dest="model_path", + default=None, + help="model path") return parser.parse_args() + def main(): options, args = option_parser() train_conf = options.train_conf @@ -119,5 +145,6 @@ def main(): predict = SentimentPrediction(train_conf, dict_file, model_path, label) predict.predict(data) + if __name__ == '__main__': main() diff --git a/demo/sentiment/preprocess.py b/demo/sentiment/preprocess.py index 49b53d500a1bf816bde9c9675b251be8e9a68ae9..7146e95d751c4de649e204fab724085994dfa4d3 100755 --- a/demo/sentiment/preprocess.py +++ b/demo/sentiment/preprocess.py @@ -22,13 +22,13 @@ from os.path import join as join_path from optparse import OptionParser from paddle.utils.preprocess_util import * - """ Usage: run following command to show help message. python preprocess.py -h """ -def save_dict(dict, filename, is_reverse = True): + +def save_dict(dict, filename, is_reverse=True): """ Save dictionary into file. dict: input dictionary. @@ -39,9 +39,10 @@ def save_dict(dict, filename, is_reverse = True): f = open(filename, 'w') for k, v in sorted(dict.items(), key=operator.itemgetter(1),\ reverse=is_reverse): - f.write('%s\t%s\n'%(k, v)) + f.write('%s\t%s\n' % (k, v)) f.close() + def tokenize(sentences): """ Use tokenizer.perl to tokenize input sentences. @@ -58,6 +59,7 @@ def tokenize(sentences): toks = tok_text.split('\n')[:-1] return toks + def read_lines(path): """ path: String, file path. @@ -71,12 +73,17 @@ def read_lines(path): seqs.append(line) return seqs + class SentimentDataSetCreate(): """ A class to process data for sentiment analysis task. """ - def __init__(self, data_path, output_path, - use_okenizer = True, multi_lines = False): + + def __init__(self, + data_path, + output_path, + use_okenizer=True, + multi_lines=False): """ data_path: string, traing and testing dataset path output_path: string, output path, store processed dataset @@ -164,23 +171,17 @@ class SentimentDataSetCreate(): # Preprocess train data. train_data, train_lab_set = self.data_list(self.train_dir) print "processing train set..." - file_lists = self.save_data(train_data, - "train", - self.batch_size, - True, - True) + file_lists = self.save_data(train_data, "train", self.batch_size, True, + True) save_list(file_lists, self.train_list) # If have test data path, preprocess test data. if os.path.exists(self.test_dir): test_data, test_lab_set = self.data_list(self.test_dir) - assert(train_lab_set == test_lab_set) + assert (train_lab_set == test_lab_set) print "processing test set..." - file_lists = self.save_data(test_data, - "test", - self.batch_size, - False, - self.dict_with_test) + file_lists = self.save_data(test_data, "test", self.batch_size, + False, self.dict_with_test) save_list(file_lists, self.test_list) # save labels set. @@ -191,7 +192,9 @@ class SentimentDataSetCreate(): save_dict(self.word_count, self.dict_file, True) self.dict_size = len(self.word_count) - def save_data(self, data, prefix = "", + def save_data(self, + data, + prefix="", batch_size=50000, is_shuffle=False, build_dict=False): @@ -205,7 +208,8 @@ class SentimentDataSetCreate(): return: list of batch names """ if is_shuffle and self.multi_lines: - return self.save_data_multi_lines(data, prefix, batch_size, build_dict) + return self.save_data_multi_lines(data, prefix, batch_size, + build_dict) if is_shuffle: random.shuffle(data) @@ -213,7 +217,7 @@ class SentimentDataSetCreate(): batch_names = [] for i in range(num_batches): batch_name = join_path(self.output_path, - "%s_part_%03d" %(prefix, i)) + "%s_part_%03d" % (prefix, i)) begin = i * batch_size end = min((i + 1) * batch_size, len(data)) # read a batch of data @@ -246,7 +250,9 @@ class SentimentDataSetCreate(): data_list = tokenize(data_list) return label_list, data_list - def save_data_multi_lines(self, data, prefix = "", + def save_data_multi_lines(self, + data, + prefix="", batch_size=50000, build_dict=False): """ @@ -274,14 +280,14 @@ class SentimentDataSetCreate(): self.create_dict(data_list) length = len(label_list) - perm_list = np.array([ i for i in xrange(length) ]) + perm_list = np.array([i for i in xrange(length)]) random.shuffle(perm_list) num_batches = int(math.ceil(length / float(batch_size))) batch_names = [] for i in range(num_batches): batch_name = join_path(self.output_path, - "%s_part_%03d" %(prefix, i)) + "%s_part_%03d" % (prefix, i)) begin = i * batch_size end = min((i + 1) * batch_size, length) sub_label = [label_list[perm_list[i]] for i in range(begin, end)] @@ -304,35 +310,50 @@ class SentimentDataSetCreate(): f.write('%s\t\t%s\n' % (lab, seq)) f.close() + def option_parser(): parser = OptionParser(usage="usage: python preprcoess.py "\ "-i data_dir [options]") - parser.add_option("-i", "--data", action="store", - dest="input", help="Input data directory.") - parser.add_option("-o", "--output", action="store", - dest="output", default=None, - help="Output directory.") - parser.add_option("-t", "--tokenizer", action="store", - dest="use_tokenizer", default=True, - help="Whether to use tokenizer.") + parser.add_option( + "-i", + "--data", + action="store", + dest="input", + help="Input data directory.") + parser.add_option( + "-o", + "--output", + action="store", + dest="output", + default=None, + help="Output directory.") + parser.add_option( + "-t", + "--tokenizer", + action="store", + dest="use_tokenizer", + default=True, + help="Whether to use tokenizer.") parser.add_option("-m", "--multi_lines", action="store", dest="multi_lines", default=False, help="If input text files have multi lines and they "\ "need to be shuffled, you should set -m True,") return parser.parse_args() + def main(): options, args = option_parser() - data_dir=options.input - output_dir=options.output - use_tokenizer=options.use_tokenizer - multi_lines=options.multi_lines + data_dir = options.input + output_dir = options.output + use_tokenizer = options.use_tokenizer + multi_lines = options.multi_lines if output_dir is None: outname = os.path.basename(options.input) output_dir = join_path(os.path.dirname(data_dir), 'pre-' + outname) - data_creator = SentimentDataSetCreate(data_dir, output_dir, - use_tokenizer, multi_lines) + data_creator = SentimentDataSetCreate(data_dir, output_dir, use_tokenizer, + multi_lines) data_creator.create_dataset() + if __name__ == '__main__': main() diff --git a/demo/sentiment/sentiment_net.py b/demo/sentiment/sentiment_net.py index 31e585edcaa111898c950ad016d3996fae15a7db..ff6a3624a404cb52d5d7ac0934fedba0d489dc22 100644 --- a/demo/sentiment/sentiment_net.py +++ b/demo/sentiment/sentiment_net.py @@ -47,10 +47,12 @@ def sentiment_data(data_dir=None, for i, line in enumerate(open(dict_file, 'r')): word_dict[line.split('\t')[0]] = i - define_py_data_sources2(train_list, test_list, - module="dataprovider", - obj="process", - args={'dictionary': word_dict}) + define_py_data_sources2( + train_list, + test_list, + module="dataprovider", + obj="process", + args={'dictionary': word_dict}) return dict_dim, class_dim @@ -64,8 +66,7 @@ def bidirectional_lstm_net(input_dim, emb = embedding_layer(input=data, size=emb_dim) bi_lstm = bidirectional_lstm(input=emb, size=lstm_dim) dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) - output = fc_layer(input=dropout, size=class_dim, - act=SoftmaxActivation()) + output = fc_layer(input=dropout, size=class_dim, act=SoftmaxActivation()) if not is_predict: lbl = data_layer("label", 1) @@ -109,27 +110,36 @@ def stacked_lstm_net(input_dim, data = data_layer("word", input_dim) emb = embedding_layer(input=data, size=emb_dim) - fc1 = fc_layer(input=emb, size=hid_dim, act=linear, - bias_attr=bias_attr) - lstm1 = lstmemory(input=fc1, act=relu, bias_attr=bias_attr, - layer_attr=layer_attr) + fc1 = fc_layer(input=emb, size=hid_dim, act=linear, bias_attr=bias_attr) + lstm1 = lstmemory( + input=fc1, act=relu, bias_attr=bias_attr, layer_attr=layer_attr) inputs = [fc1, lstm1] for i in range(2, stacked_num + 1): - fc = fc_layer(input=inputs, size=hid_dim, act=linear, - param_attr=para_attr, bias_attr=bias_attr) - lstm = lstmemory(input=fc, reverse=(i % 2) == 0, act=relu, - bias_attr=bias_attr, layer_attr=layer_attr) + fc = fc_layer( + input=inputs, + size=hid_dim, + act=linear, + param_attr=para_attr, + bias_attr=bias_attr) + lstm = lstmemory( + input=fc, + reverse=(i % 2) == 0, + act=relu, + bias_attr=bias_attr, + layer_attr=layer_attr) inputs = [fc, lstm] fc_last = pooling_layer(input=inputs[0], pooling_type=MaxPooling()) lstm_last = pooling_layer(input=inputs[1], pooling_type=MaxPooling()) - output = fc_layer(input=[fc_last, lstm_last], size=class_dim, - act=SoftmaxActivation(), - bias_attr=bias_attr, param_attr=para_attr) + output = fc_layer( + input=[fc_last, lstm_last], + size=class_dim, + act=SoftmaxActivation(), + bias_attr=bias_attr, + param_attr=para_attr) if is_predict: outputs(output) else: - outputs( - classification_cost(input=output, label=data_layer('label', 1))) + outputs(classification_cost(input=output, label=data_layer('label', 1))) diff --git a/demo/sentiment/trainer_config.py b/demo/sentiment/trainer_config.py index db24182a8d7359786bd1f3b2083892cf846605d1..894070e7c97dcb29e8c0df31437a374be5f5d691 100644 --- a/demo/sentiment/trainer_config.py +++ b/demo/sentiment/trainer_config.py @@ -20,20 +20,19 @@ is_test = get_config_arg('is_test', bool, False) # whether this config is used for prediction is_predict = get_config_arg('is_predict', bool, False) -data_dir = "./data/pre-imdb" +data_dir = "./data/pre-imdb" dict_dim, class_dim = sentiment_data(data_dir, is_test, is_predict) ################## Algorithm Config ##################### settings( - batch_size=128, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + batch_size=128, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25) #################### Network Config ###################### -stacked_lstm_net(dict_dim, class_dim=class_dim, - stacked_num=3, is_predict=is_predict) +stacked_lstm_net( + dict_dim, class_dim=class_dim, stacked_num=3, is_predict=is_predict) # bidirectional_lstm_net(dict_dim, class_dim=class_dim, is_predict=is_predict) diff --git a/demo/seqToseq/dataprovider.py b/demo/seqToseq/dataprovider.py index df19db109ed223c7515c3ebf2cb1918f41163930..c5da1b7685f47fda337921c7c60ac1497b9e48bb 100755 --- a/demo/seqToseq/dataprovider.py +++ b/demo/seqToseq/dataprovider.py @@ -30,14 +30,14 @@ def hook(settings, src_dict, trg_dict, file_list, **kwargs): if settings.job_mode: settings.trg_dict = trg_dict settings.slots = [ - integer_value_sequence(len(settings.src_dict)), - integer_value_sequence(len(settings.trg_dict)), + integer_value_sequence(len(settings.src_dict)), + integer_value_sequence(len(settings.trg_dict)), integer_value_sequence(len(settings.trg_dict)) ] settings.logger.info("trg dict len : %d" % (len(settings.trg_dict))) else: settings.slots = [ - integer_value_sequence(len(settings.src_dict)), + integer_value_sequence(len(settings.src_dict)), integer_value_sequence(len(open(file_list[0], "r").readlines())) ] @@ -62,8 +62,7 @@ def process(settings, file_name): if settings.job_mode: trg_seq = line_split[1] # one target sequence trg_words = trg_seq.split() - trg_ids = [settings.trg_dict.get(w, UNK_IDX) - for w in trg_words] + trg_ids = [settings.trg_dict.get(w, UNK_IDX) for w in trg_words] # remove sequence whose length > 80 in training mode if len(src_ids) > 80 or len(trg_ids) > 80: diff --git a/demo/seqToseq/preprocess.py b/demo/seqToseq/preprocess.py index 5efb17a664b9a2525972c29b9b5700b483b8c07e..bd1c51b1514b790ec385d48f49197b3e0285e736 100755 --- a/demo/seqToseq/preprocess.py +++ b/demo/seqToseq/preprocess.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python preprocess.py -i INPUT [-d DICTSIZE] [-m] @@ -24,12 +23,13 @@ Options: -m --mergeDict merge source and target dictionary """ import os -import sys +import sys import string from optparse import OptionParser from paddle.utils.preprocess_util import save_list, DatasetCreater + class SeqToSeqDatasetCreater(DatasetCreater): """ A class to process data for sequence to sequence application. @@ -75,7 +75,7 @@ class SeqToSeqDatasetCreater(DatasetCreater): if not os.path.exists(output): os.system(cmd + '> ' + output) - def build_dict(self, file_path, dict_path, dict_size = -1): + def build_dict(self, file_path, dict_path, dict_size=-1): """ Create the dictionary for the file, Note that 1. Valid characters include all printable characters @@ -99,20 +99,23 @@ class SeqToSeqDatasetCreater(DatasetCreater): for word in words: if word not in dictory: dictory[word] = 1 - else: + else: dictory[word] += 1 output = open(dict_path, "w+") output.write('\n\n\n') count = 3 - for key, value in sorted(dictory.items(), key = lambda d:d[1], reverse = True): + for key, value in sorted( + dictory.items(), key=lambda d: d[1], reverse=True): output.write(key + "\n") count += 1 if count == dict_size: break self.dict_size = count - - def create_dataset(self, dict_size = -1, mergeDict = False, - suffixes = ['.src', '.trg']): + + def create_dataset(self, + dict_size=-1, + mergeDict=False, + suffixes=['.src', '.trg']): """ Create seqToseq dataset """ @@ -135,13 +138,14 @@ class SeqToSeqDatasetCreater(DatasetCreater): # checkout dataset should be parallel corpora suffix_len = len(suffixes[0]) for dataset in dataset_list: - file_list = os.listdir(dataset) - if len(file_list) % 2 == 1: - raise RuntimeError("dataset should be parallel corpora") - file_list.sort() - for i in range(0, len(file_list), 2): - if file_list[i][:-suffix_len] != file_list[i + 1][:-suffix_len]: - raise RuntimeError("source and target file name should be equal") + file_list = os.listdir(dataset) + if len(file_list) % 2 == 1: + raise RuntimeError("dataset should be parallel corpora") + file_list.sort() + for i in range(0, len(file_list), 2): + if file_list[i][:-suffix_len] != file_list[i + 1][:-suffix_len]: + raise RuntimeError( + "source and target file name should be equal") # cat all the files with the same suffix in dataset for suffix in suffixes: @@ -155,16 +159,18 @@ class SeqToSeqDatasetCreater(DatasetCreater): list = ['train.list', 'test.list', 'gen.list'] for dataset in dataset_list: outname = os.path.basename(dataset) - self.concat_file(dataset, outname + suffixes[0], + self.concat_file(dataset, outname + suffixes[0], outname + suffixes[1], dir_list[id], outname) - save_list([os.path.join(dir_list[id], outname)], + save_list([os.path.join(dir_list[id], outname)], os.path.join(self.output_path, list[id])) id += 1 # build dictionary for train data dict = ['src.dict', 'trg.dict'] - dict_path = [os.path.join(self.output_path, dict[0]), - os.path.join(self.output_path, dict[1])] + dict_path = [ + os.path.join(self.output_path, dict[0]), + os.path.join(self.output_path, dict[1]) + ] if mergeDict: outname = os.path.join(train_dir, train_dataset.split('/')[-1]) print 'build src dictionary for train data' @@ -173,22 +179,30 @@ class SeqToSeqDatasetCreater(DatasetCreater): os.system('cp ' + dict_path[0] + ' ' + dict_path[1]) else: outname = os.path.join(train_dataset, self.train_dir_name) - for id in range(0,2): + for id in range(0, 2): suffix = suffixes[id] print 'build ' + suffix[1:] + ' dictionary for train data' self.build_dict(outname + suffix, dict_path[id], dict_size) print 'dictionary size is', self.dict_size + def main(): usage = "usage: \n" \ "python %prog -i INPUT [-d DICTSIZE] [-m]" parser = OptionParser(usage) - parser.add_option("-i", action="store", dest="input", - help="input original dataset path") - parser.add_option("-d", action="store", dest="dictsize", - help="specified word count of dictionary") - parser.add_option("-m", "--mergeDict", action="store_true", dest="mergeDict", - help="merge source and target dictionary") + parser.add_option( + "-i", action="store", dest="input", help="input original dataset path") + parser.add_option( + "-d", + action="store", + dest="dictsize", + help="specified word count of dictionary") + parser.add_option( + "-m", + "--mergeDict", + action="store_true", + dest="mergeDict", + help="merge source and target dictionary") (options, args) = parser.parse_args() if options.input[-1] == os.path.sep: options.input = options.input[:-1] @@ -200,5 +214,6 @@ def main(): data_creator = SeqToSeqDatasetCreater(options.input, output_path) data_creator.create_dataset(dictsize, options.mergeDict) + if __name__ == "__main__": - main(); + main() diff --git a/demo/seqToseq/seqToseq_net.py b/demo/seqToseq/seqToseq_net.py index edd6ad3f739b6cefc24d235be55c7a8f541e1ab7..ad5e3339c1461de06732eb62aca9e8323eea707b 100644 --- a/demo/seqToseq/seqToseq_net.py +++ b/demo/seqToseq/seqToseq_net.py @@ -50,16 +50,21 @@ def seq_to_seq_data(data_dir, trg_dict = None else: train_list = os.path.join(data_dir, train_list) - test_list = os.path.join(data_dir,test_list) + test_list = os.path.join(data_dir, test_list) - define_py_data_sources2(train_list, test_list, - module = "dataprovider", - obj = "process", - args = {"src_dict": src_dict, - "trg_dict": trg_dict}) + define_py_data_sources2( + train_list, + test_list, + module="dataprovider", + obj="process", + args={"src_dict": src_dict, + "trg_dict": trg_dict}) - return {"src_dict_path": src_lang_dict, "trg_dict_path": trg_lang_dict, - "gen_result": gen_result} + return { + "src_dict_path": src_lang_dict, + "trg_dict_path": trg_lang_dict, + "gen_result": gen_result + } def gru_encoder_decoder(data_conf, @@ -90,51 +95,55 @@ def gru_encoder_decoder(data_conf, size=word_vector_dim, param_attr=ParamAttr(name='_source_language_embedding')) src_forward = simple_gru(input=src_embedding, size=encoder_size) - src_backward = simple_gru(input=src_embedding, - size=encoder_size, - reverse=True) + src_backward = simple_gru( + input=src_embedding, size=encoder_size, reverse=True) encoded_vector = concat_layer(input=[src_forward, src_backward]) with mixed_layer(size=decoder_size) as encoded_proj: encoded_proj += full_matrix_projection(input=encoded_vector) backward_first = first_seq(input=src_backward) - with mixed_layer(size=decoder_size, - act=TanhActivation(), ) as decoder_boot: + with mixed_layer( + size=decoder_size, + act=TanhActivation(), ) as decoder_boot: decoder_boot += full_matrix_projection(input=backward_first) def gru_decoder_with_attention(enc_vec, enc_proj, current_word): - decoder_mem = memory(name='gru_decoder', - size=decoder_size, - boot_layer=decoder_boot) + decoder_mem = memory( + name='gru_decoder', size=decoder_size, boot_layer=decoder_boot) - context = simple_attention(encoded_sequence=enc_vec, - encoded_proj=enc_proj, - decoder_state=decoder_mem, ) + context = simple_attention( + encoded_sequence=enc_vec, + encoded_proj=enc_proj, + decoder_state=decoder_mem, ) with mixed_layer(size=decoder_size * 3) as decoder_inputs: decoder_inputs += full_matrix_projection(input=context) decoder_inputs += full_matrix_projection(input=current_word) - gru_step = gru_step_layer(name='gru_decoder', - input=decoder_inputs, - output_mem=decoder_mem, - size=decoder_size) + gru_step = gru_step_layer( + name='gru_decoder', + input=decoder_inputs, + output_mem=decoder_mem, + size=decoder_size) - with mixed_layer(size=target_dict_dim, - bias_attr=True, - act=SoftmaxActivation()) as out: + with mixed_layer( + size=target_dict_dim, bias_attr=True, + act=SoftmaxActivation()) as out: out += full_matrix_projection(input=gru_step) return out decoder_group_name = "decoder_group" - group_inputs=[StaticInput(input=encoded_vector,is_seq=True), - StaticInput(input=encoded_proj,is_seq=True)] + group_inputs = [ + StaticInput( + input=encoded_vector, is_seq=True), StaticInput( + input=encoded_proj, is_seq=True) + ] if not is_generating: trg_embedding = embedding_layer( - input=data_layer(name='target_language_word', - size=target_dict_dim), + input=data_layer( + name='target_language_word', size=target_dict_dim), size=word_vector_dim, param_attr=ParamAttr(name='_target_language_embedding')) group_inputs.append(trg_embedding) @@ -144,12 +153,12 @@ def gru_encoder_decoder(data_conf, # while encoded source sequence is accessed to as an unbounded memory. # Here, the StaticInput defines a read-only memory # for the recurrent_group. - decoder = recurrent_group(name=decoder_group_name, - step=gru_decoder_with_attention, - input=group_inputs) + decoder = recurrent_group( + name=decoder_group_name, + step=gru_decoder_with_attention, + input=group_inputs) - lbl = data_layer(name='target_language_next_word', - size=target_dict_dim) + lbl = data_layer(name='target_language_next_word', size=target_dict_dim) cost = classification_cost(input=decoder, label=lbl) outputs(cost) else: @@ -168,16 +177,19 @@ def gru_encoder_decoder(data_conf, embedding_size=word_vector_dim) group_inputs.append(trg_embedding) - beam_gen = beam_search(name=decoder_group_name, - step=gru_decoder_with_attention, - input=group_inputs, - bos_id=0, - eos_id=1, - beam_size=beam_size, - max_length=max_length) - - seqtext_printer_evaluator(input=beam_gen, - id_input=data_layer(name="sent_id", size=1), - dict_file=trg_dict_path, - result_file=gen_trans_file) + beam_gen = beam_search( + name=decoder_group_name, + step=gru_decoder_with_attention, + input=group_inputs, + bos_id=0, + eos_id=1, + beam_size=beam_size, + max_length=max_length) + + seqtext_printer_evaluator( + input=beam_gen, + id_input=data_layer( + name="sent_id", size=1), + dict_file=trg_dict_path, + result_file=gen_trans_file) outputs(beam_gen) diff --git a/demo/sequence_tagging/dataprovider.py b/demo/sequence_tagging/dataprovider.py index 6f412d6834be6d02397821215b1317353cd5df18..37dcb7aa17c0abd197ef2f3121bf8be6c54375c2 100644 --- a/demo/sequence_tagging/dataprovider.py +++ b/demo/sequence_tagging/dataprovider.py @@ -17,8 +17,7 @@ import gzip import logging logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) @@ -32,59 +31,58 @@ num_original_columns = 3 # [[-1,0], [0,0]] means previous token at column 0 and current token at # column 0 are combined as one feature. patterns = [ - [[-2,0]], - [[-1,0]], - [[0,0]], - [[1,0]], - [[2,0]], - - [[-1,0], [0,0]], - [[0,0], [1,0]], - - [[-2,1]], - [[-1,1]], - [[0,1]], - [[1,1]], - [[2,1]], - [[-2,1], [-1,1]], - [[-1,1], [0,1]], - [[0,1], [1,1]], - [[1,1], [2,1]], - - [[-2,1], [-1,1], [0,1]], - [[-1,1], [0,1], [1,1]], - [[0,1], [1,1], [2,1]], + [[-2, 0]], + [[-1, 0]], + [[0, 0]], + [[1, 0]], + [[2, 0]], + [[-1, 0], [0, 0]], + [[0, 0], [1, 0]], + [[-2, 1]], + [[-1, 1]], + [[0, 1]], + [[1, 1]], + [[2, 1]], + [[-2, 1], [-1, 1]], + [[-1, 1], [0, 1]], + [[0, 1], [1, 1]], + [[1, 1], [2, 1]], + [[-2, 1], [-1, 1], [0, 1]], + [[-1, 1], [0, 1], [1, 1]], + [[0, 1], [1, 1], [2, 1]], ] dict_label = { - 'B-ADJP': 0, - 'I-ADJP': 1, - 'B-ADVP': 2, - 'I-ADVP': 3, - 'B-CONJP': 4, - 'I-CONJP': 5, - 'B-INTJ': 6, - 'I-INTJ': 7, - 'B-LST': 8, - 'I-LST': 9, - 'B-NP': 10, - 'I-NP': 11, - 'B-PP': 12, - 'I-PP': 13, - 'B-PRT': 14, - 'I-PRT': 15, - 'B-SBAR': 16, - 'I-SBAR': 17, - 'B-UCP': 18, - 'I-UCP': 19, - 'B-VP': 20, - 'I-VP': 21, - 'O': 22 + 'B-ADJP': 0, + 'I-ADJP': 1, + 'B-ADVP': 2, + 'I-ADVP': 3, + 'B-CONJP': 4, + 'I-CONJP': 5, + 'B-INTJ': 6, + 'I-INTJ': 7, + 'B-LST': 8, + 'I-LST': 9, + 'B-NP': 10, + 'I-NP': 11, + 'B-PP': 12, + 'I-PP': 13, + 'B-PRT': 14, + 'I-PRT': 15, + 'B-SBAR': 16, + 'I-SBAR': 17, + 'B-UCP': 18, + 'I-UCP': 19, + 'B-VP': 20, + 'I-VP': 21, + 'O': 22 } + def make_features(sequence): length = len(sequence) num_features = len(sequence[0]) + def get_features(pos): if pos < 0: return ['#B%s' % -pos] * num_features @@ -94,9 +92,10 @@ def make_features(sequence): for i in xrange(length): for pattern in patterns: - fname = '/'.join([get_features(i+pos)[f] for pos, f in pattern]) + fname = '/'.join([get_features(i + pos)[f] for pos, f in pattern]) sequence[i].append(fname) + ''' Source file format: Each line is for one timestep. The features are separated by space. @@ -109,6 +108,8 @@ i-th column. return a list of dict for each column ''' + + def create_dictionaries(filename, cutoff, oov_policy): def add_to_dict(sequence, dicts): num_features = len(dicts) @@ -140,7 +141,6 @@ def create_dictionaries(filename, cutoff, oov_policy): features = line.split(' ') sequence.append(features) - for i in xrange(num_features): dct = dicts[i] n = 1 if oov_policy[i] == OOV_POLICY_USE else 0 @@ -151,7 +151,7 @@ def create_dictionaries(filename, cutoff, oov_policy): else: dct[k] = n n += 1 - + if oov_policy[i] == OOV_POLICY_USE: # placeholder so that len(dct) will be the number of features # including OOV @@ -187,12 +187,15 @@ def initializer(settings, **xargs): logger.info("feature size=%s" % dim) settings.input_types = input_types + ''' if oov_policy[i] == OOV_POLICY_USE, features in i-th column which are not existed in dicts[i] will be assigned to id 0. if oov_policy[i] == OOV_POLICY_ERROR, all features in i-th column MUST exist in dicts[i]. ''' + + @provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) def process(settings, filename): input_file = filename @@ -231,7 +234,7 @@ def process(settings, filename): logger.fatal("Unknown token: %s" % features[i]) else: vec.ids.append(dim + 0) - + dim += len(dicts[i]) sample[-1].append(vec) return sample @@ -255,4 +258,3 @@ def process(settings, filename): f.close() logger.info("num_sequences=%s" % num_sequences) - diff --git a/demo/sequence_tagging/linear_crf.py b/demo/sequence_tagging/linear_crf.py index 2bd1a20bc52fc546dcd0a0874bc09433e7212152..64895742e1b8c0a11cbedee0b88e61b5b63b007f 100644 --- a/demo/sequence_tagging/linear_crf.py +++ b/demo/sequence_tagging/linear_crf.py @@ -16,11 +16,11 @@ from paddle.trainer_config_helpers import * import math -define_py_data_sources2(train_list="data/train.list", - test_list="data/test.list", - module="dataprovider", - obj="process") - +define_py_data_sources2( + train_list="data/train.list", + test_list="data/test.list", + module="dataprovider", + obj="process") batch_size = 1 settings( @@ -30,14 +30,15 @@ settings( average_window=0.5, learning_rate=1e-1, learning_rate_decay_a=1e-5, - learning_rate_decay_b=0.25, -) + learning_rate_decay_b=0.25, ) + +num_label_types = 23 -num_label_types=23 def get_simd_size(size): return int(math.ceil(float(size) / 8)) * 8 + # Currently, in order to use sparse_update=True, # the size has to be aligned. num_label_types = get_simd_size(num_label_types) @@ -45,40 +46,37 @@ num_label_types = get_simd_size(num_label_types) features = data_layer(name="features", size=76328) word = data_layer(name="word", size=6778) pos = data_layer(name="pos", size=44) -chunk = data_layer(name="chunk", - size=num_label_types) +chunk = data_layer(name="chunk", size=num_label_types) crf_input = fc_layer( input=features, size=num_label_types, act=LinearActivation(), bias_attr=False, - param_attr=ParamAttr(initial_std=0, sparse_update=True)) + param_attr=ParamAttr( + initial_std=0, sparse_update=True)) -crf=crf_layer( +crf = crf_layer( input=crf_input, label=chunk, - param_attr=ParamAttr(name="crfw", initial_std=0), -) + param_attr=ParamAttr( + name="crfw", initial_std=0), ) -crf_decoding=crf_decoding_layer( +crf_decoding = crf_decoding_layer( size=num_label_types, input=crf_input, label=chunk, - param_attr=ParamAttr(name="crfw"), -) + param_attr=ParamAttr(name="crfw"), ) sum_evaluator( name="error", - input=crf_decoding, -) + input=crf_decoding, ) chunk_evaluator( name="chunk_f1", - input =[crf_decoding, chunk], + input=[crf_decoding, chunk], chunk_scheme="IOB", - num_chunk_types=11, -) + num_chunk_types=11, ) inputs(word, pos, chunk, features) outputs(crf) diff --git a/demo/sequence_tagging/rnn_crf.py b/demo/sequence_tagging/rnn_crf.py index fb157bf3ea7193bca2c8a281e1afaf4b5f1d7309..90d4bbdddfdb4e38b930d54a2bc865df9fac589c 100644 --- a/demo/sequence_tagging/rnn_crf.py +++ b/demo/sequence_tagging/rnn_crf.py @@ -16,10 +16,11 @@ from paddle.trainer_config_helpers import * import math -define_py_data_sources2(train_list="data/train.list", - test_list="data/test.list", - module="dataprovider", - obj="process") +define_py_data_sources2( + train_list="data/train.list", + test_list="data/test.list", + module="dataprovider", + obj="process") batch_size = 16 settings( @@ -27,29 +28,27 @@ settings( batch_size=batch_size, regularization=L2Regularization(batch_size * 1e-5), average_window=0.5, - learning_rate = 2e-3, - learning_rate_decay_a = 5e-7, - learning_rate_decay_b = 0.5, -) + learning_rate=2e-3, + learning_rate_decay_a=5e-7, + learning_rate_decay_b=0.5, ) -word_dim=128 +word_dim = 128 hidden_dim = 128 with_rnn = True -initial_std=1/math.sqrt(hidden_dim) -param_attr=ParamAttr(initial_std=initial_std) -cpu_layer_attr=ExtraLayerAttribute(device=-1) +initial_std = 1 / math.sqrt(hidden_dim) +param_attr = ParamAttr(initial_std=initial_std) +cpu_layer_attr = ExtraLayerAttribute(device=-1) default_device(0) -num_label_types=23 +num_label_types = 23 features = data_layer(name="features", size=76328) word = data_layer(name="word", size=6778) pos = data_layer(name="pos", size=44) -chunk = data_layer(name="chunk", - size=num_label_types, - layer_attr=cpu_layer_attr) +chunk = data_layer( + name="chunk", size=num_label_types, layer_attr=cpu_layer_attr) emb = embedding_layer( input=word, size=word_dim, param_attr=ParamAttr(initial_std=0)) @@ -58,73 +57,64 @@ hidden1 = mixed_layer( size=hidden_dim, act=STanhActivation(), bias_attr=True, - input=[full_matrix_projection(emb), - table_projection(pos, param_attr=param_attr)] -) + input=[ + full_matrix_projection(emb), table_projection( + pos, param_attr=param_attr) + ]) if with_rnn: rnn1 = recurrent_layer( act=ReluActivation(), bias_attr=True, input=hidden1, - param_attr=ParamAttr(initial_std=0), - ) + param_attr=ParamAttr(initial_std=0), ) hidden2 = mixed_layer( size=hidden_dim, act=STanhActivation(), bias_attr=True, - input=[full_matrix_projection(hidden1) - ] + ([ - full_matrix_projection(rnn1, param_attr=ParamAttr(initial_std=0)) - ] if with_rnn else []), -) + input=[full_matrix_projection(hidden1)] + + ([full_matrix_projection( + rnn1, param_attr=ParamAttr(initial_std=0))] if with_rnn else []), ) if with_rnn: - rnn2=recurrent_layer( + rnn2 = recurrent_layer( reverse=True, act=ReluActivation(), bias_attr=True, input=hidden2, - param_attr=ParamAttr(initial_std=0), - ) + param_attr=ParamAttr(initial_std=0), ) crf_input = mixed_layer( size=num_label_types, bias_attr=False, - input=[ - full_matrix_projection(hidden2), - ] + ([ - full_matrix_projection(rnn2, param_attr=ParamAttr(initial_std=0)) - ] if with_rnn else []), -) + input=[full_matrix_projection(hidden2), ] + + ([full_matrix_projection( + rnn2, param_attr=ParamAttr(initial_std=0))] if with_rnn else []), ) crf = crf_layer( input=crf_input, label=chunk, - param_attr=ParamAttr(name="crfw", initial_std=0), - layer_attr=cpu_layer_attr, -) + param_attr=ParamAttr( + name="crfw", initial_std=0), + layer_attr=cpu_layer_attr, ) crf_decoding = crf_decoding_layer( size=num_label_types, input=crf_input, label=chunk, param_attr=ParamAttr(name="crfw"), - layer_attr=cpu_layer_attr, -) + layer_attr=cpu_layer_attr, ) sum_evaluator( name="error", - input=crf_decoding, -) + input=crf_decoding, ) chunk_evaluator( name="chunk_f1", - input =[crf_decoding, chunk], + input=[crf_decoding, chunk], chunk_scheme="IOB", - num_chunk_types=11, -) + num_chunk_types=11, ) inputs(word, pos, chunk, features) outputs(crf) diff --git a/doc/algorithm/index.rst b/doc/algorithm/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..6073add3c0cbb12529eabb0f8d8a051bcb84e628 --- /dev/null +++ b/doc/algorithm/index.rst @@ -0,0 +1,7 @@ +Algorithm Tutorial +================== + +.. toctree:: + :maxdepth: 1 + + rnn/rnn.rst diff --git a/doc/algorithm/rnn/rnn.rst b/doc/algorithm/rnn/rnn.rst index 343f55a20e464f63f054ebe724b5ef90f848d5e9..399c5da5fffc20dda78b9eefb2629308cabd748e 100644 --- a/doc/algorithm/rnn/rnn.rst +++ b/doc/algorithm/rnn/rnn.rst @@ -1,5 +1,5 @@ -Recurrent Neural Network Configuration -====================================== +RNN Configuration +================= This tutorial will guide you how to configure recurrent neural network in PaddlePaddle. PaddlePaddle supports highly flexible and efficient recurrent neural network configuration. In this tutorial, you will learn how to: diff --git a/doc/build/contribute_to_paddle.md b/doc/build/contribute_to_paddle.md index a9ab69c5f42b8d341dca87479a642e28ca58fbf4..1d03eb7362b1b6f2fcdac7b53f8b7f93fb75e49c 100644 --- a/doc/build/contribute_to_paddle.md +++ b/doc/build/contribute_to_paddle.md @@ -1,4 +1,4 @@ -# Contribute to PaddlePaddle +# Contribute Code We sincerely appreciate your contributions. You can use fork and pull request workflow to merge your code. diff --git a/doc/build/index.rst b/doc/build/index.rst index 511cdea145c7fd0e41566d0a85115dbb06f84058..b4fe4596047c7d201fdf36bc76c26d5134611560 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -1,5 +1,5 @@ -Build And Install PaddlePaddle -================================ +Install and Build +================= Install PaddlePaddle ---------------------- @@ -18,11 +18,7 @@ Build from Source .. warning:: - Please use :code:`deb` package or :code:`docker` image to install paddle. The building guide is used for hacking or contributing to PaddlePaddle. - - -If you want to hack and contribute PaddlePaddle source code, following guides can help you\: - + Please use :code:`deb` package or :code:`docker` image to install paddle. The building guide is used for hacking or contributing PaddlePaddle source code. .. toctree:: :maxdepth: 1 @@ -30,4 +26,3 @@ If you want to hack and contribute PaddlePaddle source code, following guides ca build_from_source.md contribute_to_paddle.md - diff --git a/doc/demo/quick_start/index_en.md b/doc/demo/quick_start/index_en.md index e7d74512292c89233373c48d05895794d56702d8..80d816a768a71156ce72cda6ea92b749fbcdbe1f 100644 --- a/doc/demo/quick_start/index_en.md +++ b/doc/demo/quick_start/index_en.md @@ -1,4 +1,4 @@ -# Quick Start Tutorial +# Quick Start This tutorial will teach the basics of deep learning (DL), including how to implement many different models in PaddlePaddle. You will learn how to: - Prepare data into the standardized format that PaddlePaddle accepts. diff --git a/doc/demo/semantic_role_labeling/semantic_role_labeling.md b/doc/demo/semantic_role_labeling/semantic_role_labeling.md index 05fbc8278daf204df60ad19b742c920e47128c27..890f7314582c65e9add50664006b57aa4e0709eb 100644 --- a/doc/demo/semantic_role_labeling/semantic_role_labeling.md +++ b/doc/demo/semantic_role_labeling/semantic_role_labeling.md @@ -1,183 +1,183 @@ -# Semantic Role labeling Tutorial # - -Semantic role labeling (SRL) is a form of shallow semantic parsing whose goal is to discover the predicate-argument structure of each predicate in a given input sentence. SRL is useful as an intermediate step in a wide range of natural language processing tasks, such as information extraction. automatic document categorization and question answering. An instance is as following [1]: - - [ A0 He ] [ AM-MOD would ][ AM-NEG n’t ] [ V accept] [ A1 anything of value ] from [A2 those he was writing about ]. - -- V: verb -- A0: acceptor -- A1: thing accepted -- A2: accepted-from -- A3: Attribute -- AM-MOD: modal -- AM-NEG: negation - -Given the verb "accept", the chunks in sentence would play certain semantic roles. Here, the label scheme is from Penn Proposition Bank. - -To this date, most of the successful SRL systems are built on top of some form of parsing results where pre-defined feature templates over the syntactic structure are used. This tutorial will present an end-to-end system using deep bidirectional long short-term memory (DB-LSTM)[2] for solving the SRL task, which largely outperforms the previous state-of-the-art systems. The system regards SRL task as the sequence labelling problem. - -## Data Description -The relevant paper[2] takes the data set in CoNLL-2005&2012 Shared Task for training and testing. Accordingto data license, the demo adopts the test data set of CoNLL-2005, which can be reached on website. - -To download and process the original data, user just need to execute the following command: - -```bash -cd data -./get_data.sh -``` -Several new files appear in the `data `directory as follows. -```bash -conll05st-release：the test data set of CoNll-2005 shared task -test.wsj.words：the Wall Street Journal data sentences -test.wsj.props: the propositional arguments -src.dict：the dictionary of words in sentences -tgt.dict：the labels dictionary -feature: the extracted features from data set -``` - -## Training -### DB-LSTM -Please refer to the Sentiment Analysis demo to learn more about the long short-term memory unit. - -Unlike Bidirectional-LSTM that used in Sentiment Analysis demo, the DB-LSTM adopts another way to stack LSTM layer. First a standard LSTM processes the sequence in forward direction. The input and output of this LSTM layer are taken by the next LSTM layer as input, processed in reversed direction. These two standard LSTM layers compose a pair of LSTM. Then we stack LSTM layers pair after pair to obtain the deep LSTM model. - -The following figure shows a temporal expanded 2-layer DB-LSTM network. -
-![pic](./network_arch.png) -
- -### Features -Two input features play an essential role in this pipeline: predicate (pred) and argument (argu). Two other features: predicate context (ctx-p) and region mark (mr) are also adopted. Because a single predicate word can not exactly describe the predicate information, especially when the same words appear more than one times in a sentence. With the predicate context, the ambiguity can be largely eliminated. Similarly, we use region mark mr = 1 to denote the argument position if it locates in the predicate context region, or mr = 0 if does not. These four simple features are all we need for our SRL system. Features of one sample with context size set to 1 is showed as following[2]: -
-![pic](./feature.jpg) -
- -In this sample, the coresponding labelled sentence is: - -[ A1 A record date ] has [ AM-NEG n't ] been [ V set ] . - -In the demo, we adopt the feature template as above, consists of : `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` and use `B/I/O` scheme to label each argument. These features and labels are stored in `feature` file, and separated by `\t`. - -### Data Provider - -`dataprovider.py` is the python file to wrap data. `hook()` function is to define the data slots for network. The Six features and label are all IndexSlots. -``` -def hook(settings, word_dict, label_dict, **kwargs): - settings.word_dict = word_dict - settings.label_dict = label_dict - #all inputs are integral and sequential type - settings.slots = [ - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(2), - integer_value_sequence(len(label_dict))] -``` -The corresponding data iterator is as following: -``` -@provider(use_seq=True, init_hook=hook) -def process(obj, file_name): - with open(file_name, 'r') as fdata: - for line in fdata: - sentence, predicate, ctx_n1, ctx_0, ctx_p1, mark, label = line.strip().split('\t') - words = sentence.split() - sen_len = len(words) - word_slot = [obj.word_dict.get(w, UNK_IDX) for w in words] - - predicate_slot = [obj.word_dict.get(predicate, UNK_IDX)] * sen_len - ctx_n1_slot = [obj.word_dict.get(ctx_n1, UNK_IDX) ] * sen_len - ctx_0_slot = [obj.word_dict.get(ctx_0, UNK_IDX) ] * sen_len - ctx_p1_slot = [obj.word_dict.get(ctx_p1, UNK_IDX) ] * sen_len - - marks = mark.split() - mark_slot = [int(w) for w in marks] - - label_list = label.split() - label_slot = [obj.label_dict.get(w) for w in label_list] - - yield word_slot, predicate_slot, ctx_n1_slot, ctx_0_slot, ctx_p1_slot, mark_slot, label_slot -``` -The `process`function yield 7 lists which are six features and labels. - -### Neural Network Config -`db_lstm.py` is the neural network config file to load the dictionaries and define the data provider module and network architecture during the training procedure. - -Seven `data_layer` load instances from data provider. Six features are transformed into embedddings respectively, and mixed by `mixed_layer` . Deep bidirectional LSTM layers extract features for the softmax layer. The objective function is cross entropy of labels. - -### Run Training -The script for training is `train.sh`, user just need to execute: -```bash - ./train.sh -``` -The content in `train.sh`: -``` -paddle train \ - --config=./db_lstm.py \ - --save_dir=./output \ - --trainer_count=4 \ - --log_period=10 \ - --num_passes=500 \ - --use_gpu=false \ - --show_parameter_stats_period=10 \ - --test_all_data_in_one_period=1 \ -2>&1 | tee 'train.log' -``` - -- \--config=./db_lstm.py : network config file. -- \--save_di=./output: output path to save models. -- \--trainer_count=4 : set thread number (or GPU count). -- \--log_period=10 : print log every 20 batches. -- \--num_passes=500: set pass number, one pass in PaddlePaddle means training all samples in dataset one time. -- \--use_gpu=false: use CPU to train, set true, if you install GPU version of PaddlePaddle and want to use GPU to train. -- \--show_parameter_stats_period=10: show parameter statistic every 100 batches. -- \--test_all_data_in_one_period=1: test all data in every testing. - - -After training, the models will be saved in directory `output`. - -### Run testing -The script for testing is `test.sh`, user just need to execute: -```bash - ./test.sh -``` -The main part in `tesh.sh` -``` -paddle train \ - --config=./db_lstm.py \ - --model_list=$model_list \ - --job=test \ - --config_args=is_test=1 \ -``` - - - \--config=./db_lstm.py: network config file - - \--model_list=$model_list.list: model list file - - \--job=test: indicate the test job - - \--config_args=is_test=1: flag to indicate test - - -### Run prediction -The script for prediction is `predict.sh`, user just need to execute: -```bash - ./predict.sh - -``` -In `predict.sh`, user should offer the network config file, model path, label file, word dictionary file, feature file -``` -python predict.py - -c $config_file - -w $model_path - -l $label_file - -d $dict_file - -i $input_file -``` - -`predict.py` is the main executable python script, which includes functions: load model, load data, data prediction. The network model will output the probability distribution of labels. In the demo, we take the label with maximum probability as result. User can also implement the beam search or viterbi decoding upon the probability distribution matrix. - -After prediction, the result is saved in `predict.res`. - -## Reference -[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005. - -[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015. +# Semantic Role labeling Tutorial # + +Semantic role labeling (SRL) is a form of shallow semantic parsing whose goal is to discover the predicate-argument structure of each predicate in a given input sentence. SRL is useful as an intermediate step in a wide range of natural language processing tasks, such as information extraction. automatic document categorization and question answering. An instance is as following [1]: + + [ A0 He ] [ AM-MOD would ][ AM-NEG n’t ] [ V accept] [ A1 anything of value ] from [A2 those he was writing about ]. + +- V: verb +- A0: acceptor +- A1: thing accepted +- A2: accepted-from +- A3: Attribute +- AM-MOD: modal +- AM-NEG: negation + +Given the verb "accept", the chunks in sentence would play certain semantic roles. Here, the label scheme is from Penn Proposition Bank. + +To this date, most of the successful SRL systems are built on top of some form of parsing results where pre-defined feature templates over the syntactic structure are used. This tutorial will present an end-to-end system using deep bidirectional long short-term memory (DB-LSTM)[2] for solving the SRL task, which largely outperforms the previous state-of-the-art systems. The system regards SRL task as the sequence labelling problem. + +## Data Description +The relevant paper[2] takes the data set in CoNLL-2005&2012 Shared Task for training and testing. Accordingto data license, the demo adopts the test data set of CoNLL-2005, which can be reached on website. + +To download and process the original data, user just need to execute the following command: + +```bash +cd data +./get_data.sh +``` +Several new files appear in the `data `directory as follows. +```bash +conll05st-release：the test data set of CoNll-2005 shared task +test.wsj.words：the Wall Street Journal data sentences +test.wsj.props: the propositional arguments +src.dict：the dictionary of words in sentences +tgt.dict：the labels dictionary +feature: the extracted features from data set +``` + +## Training +### DB-LSTM +Please refer to the Sentiment Analysis demo to learn more about the long short-term memory unit. + +Unlike Bidirectional-LSTM that used in Sentiment Analysis demo, the DB-LSTM adopts another way to stack LSTM layer. First a standard LSTM processes the sequence in forward direction. The input and output of this LSTM layer are taken by the next LSTM layer as input, processed in reversed direction. These two standard LSTM layers compose a pair of LSTM. Then we stack LSTM layers pair after pair to obtain the deep LSTM model. + +The following figure shows a temporal expanded 2-layer DB-LSTM network. +
+![pic](./network_arch.png) +
+ +### Features +Two input features play an essential role in this pipeline: predicate (pred) and argument (argu). Two other features: predicate context (ctx-p) and region mark (mr) are also adopted. Because a single predicate word can not exactly describe the predicate information, especially when the same words appear more than one times in a sentence. With the predicate context, the ambiguity can be largely eliminated. Similarly, we use region mark mr = 1 to denote the argument position if it locates in the predicate context region, or mr = 0 if does not. These four simple features are all we need for our SRL system. Features of one sample with context size set to 1 is showed as following[2]: +
+![pic](./feature.jpg) +
+ +In this sample, the coresponding labelled sentence is: + +[ A1 A record date ] has [ AM-NEG n't ] been [ V set ] . + +In the demo, we adopt the feature template as above, consists of : `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` and use `B/I/O` scheme to label each argument. These features and labels are stored in `feature` file, and separated by `\t`. + +### Data Provider + +`dataprovider.py` is the python file to wrap data. `hook()` function is to define the data slots for network. The Six features and label are all IndexSlots. +``` +def hook(settings, word_dict, label_dict, **kwargs): + settings.word_dict = word_dict + settings.label_dict = label_dict + #all inputs are integral and sequential type + settings.slots = [ + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(2), + integer_value_sequence(len(label_dict))] +``` +The corresponding data iterator is as following: +``` +@provider(use_seq=True, init_hook=hook) +def process(obj, file_name): + with open(file_name, 'r') as fdata: + for line in fdata: + sentence, predicate, ctx_n1, ctx_0, ctx_p1, mark, label = line.strip().split('\t') + words = sentence.split() + sen_len = len(words) + word_slot = [obj.word_dict.get(w, UNK_IDX) for w in words] + + predicate_slot = [obj.word_dict.get(predicate, UNK_IDX)] * sen_len + ctx_n1_slot = [obj.word_dict.get(ctx_n1, UNK_IDX) ] * sen_len + ctx_0_slot = [obj.word_dict.get(ctx_0, UNK_IDX) ] * sen_len + ctx_p1_slot = [obj.word_dict.get(ctx_p1, UNK_IDX) ] * sen_len + + marks = mark.split() + mark_slot = [int(w) for w in marks] + + label_list = label.split() + label_slot = [obj.label_dict.get(w) for w in label_list] + + yield word_slot, predicate_slot, ctx_n1_slot, ctx_0_slot, ctx_p1_slot, mark_slot, label_slot +``` +The `process`function yield 7 lists which are six features and labels. + +### Neural Network Config +`db_lstm.py` is the neural network config file to load the dictionaries and define the data provider module and network architecture during the training procedure. + +Seven `data_layer` load instances from data provider. Six features are transformed into embedddings respectively, and mixed by `mixed_layer` . Deep bidirectional LSTM layers extract features for the softmax layer. The objective function is cross entropy of labels. + +### Run Training +The script for training is `train.sh`, user just need to execute: +```bash + ./train.sh +``` +The content in `train.sh`: +``` +paddle train \ + --config=./db_lstm.py \ + --save_dir=./output \ + --trainer_count=4 \ + --log_period=10 \ + --num_passes=500 \ + --use_gpu=false \ + --show_parameter_stats_period=10 \ + --test_all_data_in_one_period=1 \ +2>&1 | tee 'train.log' +``` + +- \--config=./db_lstm.py : network config file. +- \--save_di=./output: output path to save models. +- \--trainer_count=4 : set thread number (or GPU count). +- \--log_period=10 : print log every 20 batches. +- \--num_passes=500: set pass number, one pass in PaddlePaddle means training all samples in dataset one time. +- \--use_gpu=false: use CPU to train, set true, if you install GPU version of PaddlePaddle and want to use GPU to train. +- \--show_parameter_stats_period=10: show parameter statistic every 100 batches. +- \--test_all_data_in_one_period=1: test all data in every testing. + + +After training, the models will be saved in directory `output`. + +### Run testing +The script for testing is `test.sh`, user just need to execute: +```bash + ./test.sh +``` +The main part in `tesh.sh` +``` +paddle train \ + --config=./db_lstm.py \ + --model_list=$model_list \ + --job=test \ + --config_args=is_test=1 \ +``` + + - \--config=./db_lstm.py: network config file + - \--model_list=$model_list.list: model list file + - \--job=test: indicate the test job + - \--config_args=is_test=1: flag to indicate test + + +### Run prediction +The script for prediction is `predict.sh`, user just need to execute: +```bash + ./predict.sh + +``` +In `predict.sh`, user should offer the network config file, model path, label file, word dictionary file, feature file +``` +python predict.py + -c $config_file + -w $model_path + -l $label_file + -d $dict_file + -i $input_file +``` + +`predict.py` is the main executable python script, which includes functions: load model, load data, data prediction. The network model will output the probability distribution of labels. In the demo, we take the label with maximum probability as result. User can also implement the beam search or viterbi decoding upon the probability distribution matrix. + +After prediction, the result is saved in `predict.res`. + +## Reference +[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005. + +[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015. diff --git a/doc/dev/index.rst b/doc/dev/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..0468dd492b6246cfe0771a05c3597ddee95b3ddd --- /dev/null +++ b/doc/dev/index.rst @@ -0,0 +1,9 @@ +Development Guide +================= + +.. toctree:: + :maxdepth: 1 + + layer.md + new_layer/new_layer.rst + ../source/index.md diff --git a/doc/dev/layer.md b/doc/dev/layer.md new file mode 100644 index 0000000000000000000000000000000000000000..930fb0de1ac074b15d06197ed0e732f92288b411 --- /dev/null +++ b/doc/dev/layer.md @@ -0,0 +1,4 @@ +# Layer Documents + +* [Layer Source Code Document](../source/gserver/layers/index.rst) +* [Layer Python API Document](../ui/api/trainer_config_helpers/index.rst) diff --git a/doc/dev/new_layer/index.rst b/doc/dev/new_layer/index.rst deleted file mode 100644 index 37dac3a14dedf2aaa99335e1b0ebe110dc746174..0000000000000000000000000000000000000000 --- a/doc/dev/new_layer/index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Writing New Layers -================== - -.. toctree:: - :maxdepth: 3 - - new_layer.rst diff --git a/doc/dev/new_layer/new_layer.rst b/doc/dev/new_layer/new_layer.rst index bd4a4c46c87f6429338b4d220a80b6265a1f253f..2fa00730486dbe1f2c9585872068a77efa09f004 100644 --- a/doc/dev/new_layer/new_layer.rst +++ b/doc/dev/new_layer/new_layer.rst @@ -1,3 +1,4 @@ +================== Writing New Layers ================== diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..668ad75a902bdd14c6198c41380ae93e29cec0d3 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,10 @@ +PaddlePaddle Documentation +========================== + +.. toctree:: + :maxdepth: 1 + + introduction/index.md + user_guide.rst + dev/index.rst + algorithm/index.rst diff --git a/doc/introduction/index.md b/doc/introduction/index.md index 004ca07844da0fdbea359508c9fae1012aaad421..01f52031a1d0247cd0b885218c17001f23685239 100644 --- a/doc/introduction/index.md +++ b/doc/introduction/index.md @@ -98,4 +98,3 @@ There, you have recovered the underlying pattern between `X` and `Y` only from o - Build and Installation - Quick Start - Example and Demo - diff --git a/doc/layer.md b/doc/layer.md deleted file mode 100644 index 45f2e2bad542ff5c29c89201b356728cf7ca8c1c..0000000000000000000000000000000000000000 --- a/doc/layer.md +++ /dev/null @@ -1,4 +0,0 @@ -# Layer Documents - -* [Layer Source Code Document](source/gserver/layers/index.rst) -* [Layer Python API Document](ui/api/trainer_config_helpers/layers_index.rst) diff --git a/doc/source/gserver/layers/layer.rst b/doc/source/gserver/layers/layer.rst index 807b22ca140ee71208a96e2877b9c5636620b165..4b8e149505f0695ad2fa4be967a50d1a0ac48b43 100644 --- a/doc/source/gserver/layers/layer.rst +++ b/doc/source/gserver/layers/layer.rst @@ -465,6 +465,11 @@ SumOfSquaresCostLayer .. doxygenclass:: paddle::SumOfSquaresCostLayer :members: +SumCostLayer +````````````````````` +.. doxygenclass:: paddle::SumCostLayer + :members: + CosSimLayer ----------- .. doxygenclass:: paddle::CosSimLayer diff --git a/doc/ui/api/trainer_config_helpers/activations.rst b/doc/ui/api/trainer_config_helpers/activations.rst index 070ed03ab6cc938f735667701bd46eec33ea77b4..269e6491e7ebe3899c3fb24fca756a393043473b 100644 --- a/doc/ui/api/trainer_config_helpers/activations.rst +++ b/doc/ui/api/trainer_config_helpers/activations.rst @@ -1,3 +1,7 @@ +=========== +Activations +=========== + BaseActivation ============== @@ -102,4 +106,3 @@ STanhActivation .. automodule:: paddle.trainer_config_helpers.activations :members: STanhActivation :noindex: - diff --git a/doc/ui/api/trainer_config_helpers/activations_index.rst b/doc/ui/api/trainer_config_helpers/activations_index.rst deleted file mode 100644 index 1c0b71ab77eec62859c1d7615f6ebe637f3108ac..0000000000000000000000000000000000000000 --- a/doc/ui/api/trainer_config_helpers/activations_index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Activations -=========== - -.. toctree:: - :maxdepth: 3 - - activations.rst diff --git a/doc/ui/api/trainer_config_helpers/evaluators.rst b/doc/ui/api/trainer_config_helpers/evaluators.rst index 0586c9907e472dd98c5f7e9098251f3bc6b88bab..d6a79c13e2316b0fd3d53eb47960a767bcf8abdb 100644 --- a/doc/ui/api/trainer_config_helpers/evaluators.rst +++ b/doc/ui/api/trainer_config_helpers/evaluators.rst @@ -1,3 +1,7 @@ +========== +Evaluators +========== + Base ==== .. automodule:: paddle.trainer_config_helpers.evaluators diff --git a/doc/ui/api/trainer_config_helpers/evaluators_index.rst b/doc/ui/api/trainer_config_helpers/evaluators_index.rst deleted file mode 100644 index 298de3e1a32d36b9102f5ad64cc1b968f418041b..0000000000000000000000000000000000000000 --- a/doc/ui/api/trainer_config_helpers/evaluators_index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Evaluators -========== - -.. toctree:: - :maxdepth: 3 - - evaluators.rst diff --git a/doc/ui/api/trainer_config_helpers/index.md b/doc/ui/api/trainer_config_helpers/index.md deleted file mode 100644 index 00fa99bb3fa4c407dc867f91f4c7c495dc4061a1..0000000000000000000000000000000000000000 --- a/doc/ui/api/trainer_config_helpers/index.md +++ /dev/null @@ -1,10 +0,0 @@ -# Model Config Interface - -* [Optimizer](optimizers_index.rst) -* [Data Source](data_sources.rst) -* [Layers](layers_index.rst) -* [Activations](activations_index.rst) -* [Poolings](poolings_index.rst) -* [Networks](networks_index.rst) -* [Evaluators](evaluators_index.rst) -* [Parameter and Extra Layer Attribute](attrs.rst) diff --git a/doc/ui/api/trainer_config_helpers/index.rst b/doc/ui/api/trainer_config_helpers/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..8395eb75710b3e67ec0c5442f79c999bdacdff42 --- /dev/null +++ b/doc/ui/api/trainer_config_helpers/index.rst @@ -0,0 +1,14 @@ +Model Config Interface +====================== + +.. toctree:: + :maxdepth: 1 + + optimizers.rst + data_sources.rst + layers.rst + activations.rst + poolings.rst + networks.rst + evaluators.rst + attrs.rst diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index c78682423e448a472ca46f2cb100a40efface6eb..b487b739a719e9f7118efcc143301da36f7a978e 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -1,3 +1,7 @@ +====== +Layers +====== + Base ====== @@ -46,6 +50,12 @@ conv_operator :members: conv_operator :noindex: +conv_projection +--------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: conv_projection + :noindex: + conv_shift_layer ------------------ .. automodule:: paddle.trainer_config_helpers.layers @@ -71,6 +81,12 @@ img_pool_layer -------------- .. automodule:: paddle.trainer_config_helpers.layers :members: img_pool_layer + :noindex: + +spp_layer +-------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: spp_layer :noindex: maxout_layer @@ -175,6 +191,12 @@ embedding_layer :members: embedding_layer :noindex: +scaling_projection +------------------ +.. automodule:: paddle.trainer_config_helpers.layers + :members: scaling_projection + :noindex: + dotmul_projection ----------------- .. automodule:: paddle.trainer_config_helpers.layers @@ -254,6 +276,12 @@ expand_layer :members: expand_layer :noindex: +repeat_layer +------------ +.. automodule:: paddle.trainer_config_helpers.layers + :members: repeat_layer + :noindex: + Math Layers =========== @@ -401,6 +429,12 @@ hsigmoid :members: hsigmoid :noindex: +sum_cost +--------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: sum_cost + :noindex: + Check Layer ============ diff --git a/doc/ui/api/trainer_config_helpers/layers_index.rst b/doc/ui/api/trainer_config_helpers/layers_index.rst deleted file mode 100644 index c0daab152148ce769948f600c3101bd79f5a1013..0000000000000000000000000000000000000000 --- a/doc/ui/api/trainer_config_helpers/layers_index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Layers -====== - -.. toctree:: - :maxdepth: 3 - - layers.rst diff --git a/doc/ui/api/trainer_config_helpers/networks.rst b/doc/ui/api/trainer_config_helpers/networks.rst index 2a15b34eaea0b763f992a7225550e6af747f303c..29c52c5ce3078f1755162dbbdd65a059d8ba9fa4 100644 --- a/doc/ui/api/trainer_config_helpers/networks.rst +++ b/doc/ui/api/trainer_config_helpers/networks.rst @@ -1,3 +1,9 @@ +======== +Networks +======== + +The networks module contains pieces of neural network that combine multiple layers. + NLP === @@ -111,4 +117,3 @@ outputs .. automodule:: paddle.trainer_config_helpers.networks :members: outputs :noindex: - diff --git a/doc/ui/api/trainer_config_helpers/networks_index.rst b/doc/ui/api/trainer_config_helpers/networks_index.rst deleted file mode 100644 index 17bc4dfaa6c4ed3cd5daf0476d0d4c15a2067a22..0000000000000000000000000000000000000000 --- a/doc/ui/api/trainer_config_helpers/networks_index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Networks -======== - -The networks module contains pieces of neural network that combine multiple layers. - -.. toctree:: - :maxdepth: 3 - - networks.rst diff --git a/doc/ui/api/trainer_config_helpers/optimizers.rst b/doc/ui/api/trainer_config_helpers/optimizers.rst index b487fec64c4ebb5cfbdff1aa101d9b3675776a2c..7ca4e34156e273caf66cc71e6927bfb23bb5235e 100644 --- a/doc/ui/api/trainer_config_helpers/optimizers.rst +++ b/doc/ui/api/trainer_config_helpers/optimizers.rst @@ -1,3 +1,7 @@ +========== +Optimizers +========== + BaseSGDOptimizer ================ .. automodule:: paddle.trainer_config_helpers.optimizers @@ -51,4 +55,3 @@ settings .. automodule:: paddle.trainer_config_helpers.optimizers :members: settings :noindex: - diff --git a/doc/ui/api/trainer_config_helpers/optimizers_index.rst b/doc/ui/api/trainer_config_helpers/optimizers_index.rst deleted file mode 100644 index f39f94f0cd6e1a6c3c25eeceb7820a7fbc070570..0000000000000000000000000000000000000000 --- a/doc/ui/api/trainer_config_helpers/optimizers_index.rst +++ /dev/null @@ -1,7 +0,0 @@ -Optimizers -========== - -.. toctree:: - :maxdepth: 3 - - optimizers.rst diff --git a/doc/ui/api/trainer_config_helpers/poolings.rst b/doc/ui/api/trainer_config_helpers/poolings.rst index caadec639383aad24ed477d8bdaeaa31c0026bb5..66566809d26f59263597b5286c5b27e0bbc9415a 100644 --- a/doc/ui/api/trainer_config_helpers/poolings.rst +++ b/doc/ui/api/trainer_config_helpers/poolings.rst @@ -1,3 +1,7 @@ +======== +Poolings +======== + BasePoolingType =============== .. automodule:: paddle.trainer_config_helpers.poolings @@ -27,4 +31,3 @@ SquareRootNPooling .. automodule:: paddle.trainer_config_helpers.poolings :members: SquareRootNPooling :noindex: - diff --git a/doc/ui/api/trainer_config_helpers/poolings_index.rst b/doc/ui/api/trainer_config_helpers/poolings_index.rst deleted file mode 100644 index 250d3fa69c0dcedfd689b685fe7b47ec71d02fee..0000000000000000000000000000000000000000 --- a/doc/ui/api/trainer_config_helpers/poolings_index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Poolings -======== - -These pooling types are used for sequence input, not for images. - -.. toctree:: - :maxdepth: 3 - - poolings.rst diff --git a/doc/ui/predict/predict_sample.py b/doc/ui/predict/predict_sample.py index d55d2c730dece07f068b728d0a75f34c70b817bd..63e8b36d26057d4a87dabb8745de8e13efe2524f 100644 --- a/doc/ui/predict/predict_sample.py +++ b/doc/ui/predict/predict_sample.py @@ -16,82 +16,113 @@ from py_paddle import swig_paddle, DataProviderConverter from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -TEST_DATA = [[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.215686, - 0.533333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.67451, - 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.886275, - 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.192157, 0.070588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.670588, 0.992157, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.117647, 0.933333, 0.858824, 0.313725, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0.090196, 0.858824, 0.992157, 0.831373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.141176, - 0.992157, 0.992157, 0.611765, 0.054902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.258824, 0.992157, 0.992157, - 0.529412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.368627, 0.992157, 0.992157, 0.419608, 0.003922, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0.094118, 0.835294, 0.992157, 0.992157, 0.517647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.603922, 0.992157, - 0.992157, 0.992157, 0.603922, 0.545098, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0.447059, 0.992157, 0.992157, - 0.956863, 0.062745, 0, 0, 0, 0, 0, 0, 0, 0, 0.011765, 0.666667, 0.992157, 0.992157, 0.992157, 0.992157, - 0.992157, 0.745098, 0.137255, 0, 0, 0, 0, 0, 0.152941, 0.866667, 0.992157, 0.992157, 0.521569, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, 0.992157, 0.992157, 0.803922, 0.352941, 0.745098, 0.992157, - 0.945098, 0.317647, 0, 0, 0, 0, 0.580392, 0.992157, 0.992157, 0.764706, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0.070588, 0.992157, 0.992157, 0.776471, 0.043137, 0, 0.007843, 0.27451, 0.882353, 0.941176, 0.176471, - 0, 0, 0.180392, 0.898039, 0.992157, 0.992157, 0.313725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, - 0.992157, 0.713725, 0, 0, 0, 0, 0.627451, 0.992157, 0.729412, 0.062745, 0, 0.509804, 0.992157, 0.992157, - 0.776471, 0.035294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.494118, 0.992157, 0.992157, 0.968627, 0.168627, 0, 0, - 0, 0.423529, 0.992157, 0.992157, 0.364706, 0, 0.717647, 0.992157, 0.992157, 0.317647, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0.533333, 0.992157, 0.984314, 0.945098, 0.603922, 0, 0, 0, 0.003922, 0.466667, 0.992157, - 0.988235, 0.976471, 0.992157, 0.992157, 0.788235, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.686275, - 0.882353, 0.364706, 0, 0, 0, 0, 0, 0, 0.098039, 0.588235, 0.992157, 0.992157, 0.992157, 0.980392, - 0.305882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.101961, 0.67451, 0.321569, 0, 0, 0, 0, 0, 0, 0, 0.105882, - 0.733333, 0.976471, 0.811765, 0.713725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.65098, 0.992157, - 0.321569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.25098, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0.94902, 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.968627, - 0.764706, 0.152941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.498039, - 0.25098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.298039, 0.333333, 0.333333, 0.333333, 0.337255, 0.333333, - 0.333333, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.027451, 0.223529, 0.776471, - 0.964706, 0.988235, 0.988235, 0.988235, 0.992157, 0.988235, 0.988235, 0.780392, 0.098039, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.14902, 0.698039, 0.988235, 0.992157, 0.988235, 0.901961, 0.87451, - 0.568627, 0.882353, 0.976471, 0.988235, 0.988235, 0.501961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.188235, 0.647059, 0.988235, 0.988235, 0.745098, 0.439216, 0.098039, 0, 0, 0, 0.572549, 0.988235, - 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, 0.933333, 0.992157, 0.941176, - 0.247059, 0, 0, 0, 0, 0, 0, 0.188235, 0.898039, 0.992157, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.039216, 0.639216, 0.933333, 0.988235, 0.913725, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0.113725, 0.843137, - 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.235294, 0.988235, 0.992157, 0.988235, 0.815686, - 0.07451, 0, 0, 0, 0, 0, 0, 0, 0.333333, 0.988235, 0.988235, 0.552941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.211765, 0.878431, 0.988235, 0.992157, 0.701961, 0.329412, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0.698039, - 0.988235, 0.913725, 0.145098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.890196, 0.988235, 0.988235, - 0.745098, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.882353, 0.988235, 0.568627, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0.2, 0.933333, 0.992157, 0.992157, 0.992157, 0.447059, 0.294118, 0, 0, 0, 0, 0, 0, 0, 0, 0.447059, - 0.992157, 0.768627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.988235, 0.988235, 0.988235, - 0.992157, 0.47451, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.933333, 0.87451, 0.509804, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0.992157, 0.988235, 0.937255, 0.792157, 0.988235, 0.894118, 0.082353, 0, 0, 0, 0, 0, 0, - 0.027451, 0.647059, 0.992157, 0.654902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.913725, - 0.329412, 0.376471, 0.184314, 0, 0, 0, 0, 0, 0, 0.027451, 0.513725, 0.988235, 0.635294, 0.219608, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.929412, 0.988235, 0.988235, 0.741176, 0.309804, 0, 0, 0, 0, - 0, 0, 0.529412, 0.988235, 0.678431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.223529, 0.992157, - 0.992157, 1, 0.992157, 0.992157, 0.992157, 0.992157, 1, 0.992157, 0.992157, 0.882353, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.023529, 0.478431, 0.654902, 0.658824, 0.952941, 0.988235, 0.988235, - 0.988235, 0.992157, 0.988235, 0.729412, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0.196078, 0.647059, 0.764706, 0.764706, 0.768627, 0.580392, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0]]] +TEST_DATA = [[[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.215686, 0.533333, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.67451, 0.992157, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.070588, 0.886275, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.192157, + 0.070588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.670588, 0.992157, + 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.117647, 0.933333, 0.858824, 0.313725, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.090196, 0.858824, 0.992157, 0.831373, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.141176, 0.992157, 0.992157, 0.611765, 0.054902, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.258824, 0.992157, 0.992157, 0.529412, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.368627, 0.992157, 0.992157, 0.419608, 0.003922, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.094118, 0.835294, 0.992157, 0.992157, 0.517647, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.603922, 0.992157, 0.992157, 0.992157, 0.603922, + 0.545098, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0.447059, 0.992157, 0.992157, + 0.956863, 0.062745, 0, 0, 0, 0, 0, 0, 0, 0, 0.011765, 0.666667, 0.992157, + 0.992157, 0.992157, 0.992157, 0.992157, 0.745098, 0.137255, 0, 0, 0, 0, 0, + 0.152941, 0.866667, 0.992157, 0.992157, 0.521569, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.070588, 0.992157, 0.992157, 0.992157, 0.803922, 0.352941, 0.745098, + 0.992157, 0.945098, 0.317647, 0, 0, 0, 0, 0.580392, 0.992157, 0.992157, + 0.764706, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, 0.992157, + 0.776471, 0.043137, 0, 0.007843, 0.27451, 0.882353, 0.941176, 0.176471, 0, + 0, 0.180392, 0.898039, 0.992157, 0.992157, 0.313725, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0.070588, 0.992157, 0.992157, 0.713725, 0, 0, 0, 0, 0.627451, + 0.992157, 0.729412, 0.062745, 0, 0.509804, 0.992157, 0.992157, 0.776471, + 0.035294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.494118, 0.992157, 0.992157, + 0.968627, 0.168627, 0, 0, 0, 0.423529, 0.992157, 0.992157, 0.364706, 0, + 0.717647, 0.992157, 0.992157, 0.317647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.533333, 0.992157, 0.984314, 0.945098, 0.603922, 0, 0, 0, 0.003922, + 0.466667, 0.992157, 0.988235, 0.976471, 0.992157, 0.992157, 0.788235, + 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.686275, 0.882353, 0.364706, 0, + 0, 0, 0, 0, 0, 0.098039, 0.588235, 0.992157, 0.992157, 0.992157, 0.980392, + 0.305882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.101961, 0.67451, 0.321569, + 0, 0, 0, 0, 0, 0, 0, 0.105882, 0.733333, 0.976471, 0.811765, 0.713725, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.65098, 0.992157, 0.321569, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0.25098, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 0.94902, 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.968627, 0.764706, 0.152941, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.498039, 0.25098, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 +]], [[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.298039, 0.333333, 0.333333, 0.333333, 0.337255, + 0.333333, 0.333333, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0.027451, 0.223529, 0.776471, 0.964706, 0.988235, 0.988235, 0.988235, + 0.992157, 0.988235, 0.988235, 0.780392, 0.098039, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.14902, 0.698039, 0.988235, 0.992157, 0.988235, 0.901961, + 0.87451, 0.568627, 0.882353, 0.976471, 0.988235, 0.988235, 0.501961, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.647059, 0.988235, 0.988235, + 0.745098, 0.439216, 0.098039, 0, 0, 0, 0.572549, 0.988235, 0.988235, + 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, 0.933333, 0.992157, + 0.941176, 0.247059, 0, 0, 0, 0, 0, 0, 0.188235, 0.898039, 0.992157, + 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.039216, 0.639216, 0.933333, + 0.988235, 0.913725, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0.113725, 0.843137, + 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.235294, 0.988235, + 0.992157, 0.988235, 0.815686, 0.07451, 0, 0, 0, 0, 0, 0, 0, 0.333333, + 0.988235, 0.988235, 0.552941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.211765, + 0.878431, 0.988235, 0.992157, 0.701961, 0.329412, 0.109804, 0, 0, 0, 0, 0, + 0, 0, 0.698039, 0.988235, 0.913725, 0.145098, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.188235, 0.890196, 0.988235, 0.988235, 0.745098, 0.047059, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0.882353, 0.988235, 0.568627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, + 0.933333, 0.992157, 0.992157, 0.992157, 0.447059, 0.294118, 0, 0, 0, 0, 0, + 0, 0, 0, 0.447059, 0.992157, 0.768627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.623529, 0.988235, 0.988235, 0.988235, 0.988235, 0.992157, 0.47451, 0, 0, + 0, 0, 0, 0, 0, 0.188235, 0.933333, 0.87451, 0.509804, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0.992157, 0.988235, 0.937255, 0.792157, 0.988235, 0.894118, + 0.082353, 0, 0, 0, 0, 0, 0, 0.027451, 0.647059, 0.992157, 0.654902, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.913725, 0.329412, 0.376471, + 0.184314, 0, 0, 0, 0, 0, 0, 0.027451, 0.513725, 0.988235, 0.635294, + 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.929412, 0.988235, + 0.988235, 0.741176, 0.309804, 0, 0, 0, 0, 0, 0, 0.529412, 0.988235, + 0.678431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.223529, 0.992157, + 0.992157, 1, 0.992157, 0.992157, 0.992157, 0.992157, 1, 0.992157, 0.992157, + 0.882353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.023529, + 0.478431, 0.654902, 0.658824, 0.952941, 0.988235, 0.988235, 0.988235, + 0.992157, 0.988235, 0.729412, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.647059, 0.764706, 0.764706, 0.768627, + 0.580392, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0 +]]] def main(): conf = parse_config("./mnist_model/trainer_config.py", "") print conf.data_config.load_data_args - network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) # For code hint. network.loadParameters("./mnist_model/") converter = DataProviderConverter([dense_vector(784)]) diff --git a/doc/user_guide.rst b/doc/user_guide.rst new file mode 100644 index 0000000000000000000000000000000000000000..d4deb3ca5a4523b509ea5082f32be8a315570dea --- /dev/null +++ b/doc/user_guide.rst @@ -0,0 +1,13 @@ +User Guide +========== + +.. toctree:: + :maxdepth: 1 + + demo/quick_start/index_en.md + build/index.rst + build/contribute_to_paddle.md + ui/index.md + ui/api/trainer_config_helpers/index.rst + demo/index.md + cluster/index.md diff --git a/doc_cn/algorithm/rnn/hierarchical-layer.md b/doc_cn/algorithm/rnn/hierarchical-layer.md index 5282bbbcb82d00f5aed7b784d2bd44f9ec33fa42..519653df081d6e7919ada3cbff6aaf4d2a2f6115 100644 --- a/doc_cn/algorithm/rnn/hierarchical-layer.md +++ b/doc_cn/algorithm/rnn/hierarchical-layer.md @@ -1,66 +1,66 @@ -# 支持双层序列作为输入的Layer - -## 概述 - -在自然语言处理任务中，序列是一种常见的数据类型。一个独立的词语，可以看作是一个非序列输入，或者，我们称之为一个0层的序列；由词语构成的句子，是一个单层序列；若干个句子构成一个段落，是一个双层的序列。 - -双层序列是一个嵌套的序列，它的每一个元素，又是一个单层的序列。这是一种非常灵活的数据组织方式，帮助我们构造一些复杂的输入信息。 - -我们可以按照如下层次定义非序列，单层序列，以及双层序列。 - -+ 0层序列：一个独立的元素，类型可以是PaddlePaddle支持的任意输入数据类型 -+ 单层序列：排成一列的多个元素，每个元素是一个0层序列，元素之间的顺序是重要的输入信息 -+ 双层序列：排成一列的多个元素，每个元素是一个单层序列，称之为双层序列的一个子序列（subseq），subseq的每个元素是一个0层序列 - - -在 PaddlePaddle中，下面这些Layer能够接受双层序列作为输入，完成相应的计算。 -## pooling_layer - -pooling_layer的使用示例如下，详细见配置API。 -```python -seq_pool = pooling_layer(input=layer, - pooling_type=AvgPooling(), - agg_level=AggregateLevel.EACH_SEQUENCE) -``` -- `pooling_type` 目前支持两种，分别是：MaxPooling()和AvgPooling()。 -- `agg_level=AggregateLevel.TIMESTEP`时（默认值）： - - 作用：双层序列经过运算变成一个0层序列，或单层序列经过运算变成一个0层序列 - - 输入：一个双层序列，或一个单层序列 - - 输出：一个0层序列，即整个输入序列（单层或双层）的平均值（或最大值） -- `agg_level=AggregateLevel.EACH_SEQUENCE`时： - - 作用：一个双层序列经过运算变成一个单层序列 - - 输入：必须是一个双层序列 - - 输出：一个单层序列，序列的每个元素是原来双层序列每个subseq元素的平均值（或最大值） - -## last_seq 和 first_seq - -last_seq的使用示例如下（first_seq类似），详细见配置API。 -```python -last = last_seq(input=layer, - agg_level=AggregateLevel.EACH_SEQUENCE) -``` -- `agg_level=AggregateLevel.TIMESTEP`时（默认值）： - - 作用：一个双层序列经过运算变成一个0层序列，或一个单层序列经过运算变成一个0层序列 - - 输入：一个双层序列或一个单层序列 - - 输出：一个0层序列，即整个输入序列（双层或者单层）最后一个，或第一个元素。 -- `agg_level=AggregateLevel.EACH_SEQUENCE`时： - - 作用：一个双层序列经过运算变成一个单层序列 - - 输入：必须是一个双层序列 - - 输出：一个单层序列，其中每个元素是双层序列中每个subseq最后一个（或第一个）元素。 - -## expand_layer - -expand_layer的使用示例如下，详细见配置API。 -```python -expand = expand_layer(input=layer1, - expand_as=layer2, - expand_level=ExpandLevel.FROM_TIMESTEP) -``` -- `expand_level=ExpandLevel.FROM_TIMESTEP`时（默认值）： - - 作用：一个0层序列经过运算扩展成一个单层序列，或者一个双层序列 - - 输入：layer1必须是一个0层序列，是待扩展的数据；layer2可以是一个单层序列，或者是一个双层序列，提供扩展的长度信息 - - 输出：一个单层序列，或一个双层序列，输出序列的类型（双层序列，或单层序列）和序列中含有元素的数目同 layer2一致。若输出是单层序列，单层序列的每个元素（0层序列），都是对layer1元素的拷贝；若输出是双层序列，双层序列每个subseq中每个元素（0层序列），都是对layer1元素的拷贝 -- `expand_level=ExpandLevel.FROM_SEQUENCE`时： - - 作用：一个单层序列经过运算扩展成一个双层序列 - - 输入：layer1必须是一个单层序列，是待扩展的数据；layer2必须是一个双层序列，提供扩展的长度信息 - - 输出：一个双层序列，序列中含有元素的数目同layer2一致。要求单层序列含有元素的数目（0层序列），和双层序列含有subseq 的数目一致。单层序列第i个元素（0层序列），被扩展为一个单层序列，构成了输出双层序列的第i个subseq。 \ No newline at end of file +# 支持双层序列作为输入的Layer + +## 概述 + +在自然语言处理任务中，序列是一种常见的数据类型。一个独立的词语，可以看作是一个非序列输入，或者，我们称之为一个0层的序列；由词语构成的句子，是一个单层序列；若干个句子构成一个段落，是一个双层的序列。 + +双层序列是一个嵌套的序列，它的每一个元素，又是一个单层的序列。这是一种非常灵活的数据组织方式，帮助我们构造一些复杂的输入信息。 + +我们可以按照如下层次定义非序列，单层序列，以及双层序列。 + ++ 0层序列：一个独立的元素，类型可以是PaddlePaddle支持的任意输入数据类型 ++ 单层序列：排成一列的多个元素，每个元素是一个0层序列，元素之间的顺序是重要的输入信息 ++ 双层序列：排成一列的多个元素，每个元素是一个单层序列，称之为双层序列的一个子序列（subseq），subseq的每个元素是一个0层序列 + + +在 PaddlePaddle中，下面这些Layer能够接受双层序列作为输入，完成相应的计算。 +## pooling_layer + +pooling_layer的使用示例如下，详细见配置API。 +```python +seq_pool = pooling_layer(input=layer, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) +``` +- `pooling_type` 目前支持两种，分别是：MaxPooling()和AvgPooling()。 +- `agg_level=AggregateLevel.TIMESTEP`时（默认值）： + - 作用：双层序列经过运算变成一个0层序列，或单层序列经过运算变成一个0层序列 + - 输入：一个双层序列，或一个单层序列 + - 输出：一个0层序列，即整个输入序列（单层或双层）的平均值（或最大值） +- `agg_level=AggregateLevel.EACH_SEQUENCE`时： + - 作用：一个双层序列经过运算变成一个单层序列 + - 输入：必须是一个双层序列 + - 输出：一个单层序列，序列的每个元素是原来双层序列每个subseq元素的平均值（或最大值） + +## last_seq 和 first_seq + +last_seq的使用示例如下（first_seq类似），详细见配置API。 +```python +last = last_seq(input=layer, + agg_level=AggregateLevel.EACH_SEQUENCE) +``` +- `agg_level=AggregateLevel.TIMESTEP`时（默认值）： + - 作用：一个双层序列经过运算变成一个0层序列，或一个单层序列经过运算变成一个0层序列 + - 输入：一个双层序列或一个单层序列 + - 输出：一个0层序列，即整个输入序列（双层或者单层）最后一个，或第一个元素。 +- `agg_level=AggregateLevel.EACH_SEQUENCE`时： + - 作用：一个双层序列经过运算变成一个单层序列 + - 输入：必须是一个双层序列 + - 输出：一个单层序列，其中每个元素是双层序列中每个subseq最后一个（或第一个）元素。 + +## expand_layer + +expand_layer的使用示例如下，详细见配置API。 +```python +expand = expand_layer(input=layer1, + expand_as=layer2, + expand_level=ExpandLevel.FROM_TIMESTEP) +``` +- `expand_level=ExpandLevel.FROM_TIMESTEP`时（默认值）： + - 作用：一个0层序列经过运算扩展成一个单层序列，或者一个双层序列 + - 输入：layer1必须是一个0层序列，是待扩展的数据；layer2可以是一个单层序列，或者是一个双层序列，提供扩展的长度信息 + - 输出：一个单层序列，或一个双层序列，输出序列的类型（双层序列，或单层序列）和序列中含有元素的数目同 layer2一致。若输出是单层序列，单层序列的每个元素（0层序列），都是对layer1元素的拷贝；若输出是双层序列，双层序列每个subseq中每个元素（0层序列），都是对layer1元素的拷贝 +- `expand_level=ExpandLevel.FROM_SEQUENCE`时： + - 作用：一个单层序列经过运算扩展成一个双层序列 + - 输入：layer1必须是一个单层序列，是待扩展的数据；layer2必须是一个双层序列，提供扩展的长度信息 + - 输出：一个双层序列，序列中含有元素的数目同layer2一致。要求单层序列含有元素的数目（0层序列），和双层序列含有subseq 的数目一致。单层序列第i个元素（0层序列），被扩展为一个单层序列，构成了输出双层序列的第i个subseq。 diff --git a/doc_cn/algorithm/rnn/hierarchical-rnn.md b/doc_cn/algorithm/rnn/hierarchical-rnn.md index 4a85cf336146ef368b04c13fdc74f39ee7a361d3..c184a34e85a571e98e88c14ef653356fdd555a19 100644 --- a/doc_cn/algorithm/rnn/hierarchical-rnn.md +++ b/doc_cn/algorithm/rnn/hierarchical-rnn.md @@ -1,403 +1,403 @@ -# 双层RNN配置与示例 - -我们在`paddle/gserver/tests/test_RecurrentGradientMachine`单测中，通过多组语义相同的单双层RNN配置，讲解如何使用双层RNN。 - -## 示例1：双进双出，subseq间无memory - -配置：单层RNN（`sequence_layer_group`）和双层RNN（`sequence_nest_layer_group`），语义完全相同。 - -### 读取双层序列的方法 - -首先，我们看一下单双层序列的不同数据组织形式（您也可以采用别的组织形式）： - -- 单层序列的数据（`Sequence/tour_train_wdseg`）如下，一共有10个样本。每个样本由两部分组成，一个label（此处都为2）和一个已经分词后的句子。 - -```text -2 酒店 有 很 舒适 的 床垫 子 ， 床上用品 也 应该 是 一人 一 换 ， 感觉 很 利落 对 卫生 很 放心 呀 。 -2 很 温馨 ， 也 挺 干净 的 * 地段 不错 ， 出来 就 有 全家 ， 离 地铁站 也 近 ， 交通 很方便 * 就是 都 不 给 刷牙 的 杯子 啊 ， 就 第一天 给 了 一次性杯子 * -2 位置 方便 ， 强烈推荐 ， 十一 出去玩 的 时候 选 的 ， 对面 就是 华润万家 ， 周围 吃饭 的 也 不少 。 -2 交通便利 ， 吃 很 便利 ， 乾 浄 、 安静 ， 商务 房 有 电脑 、 上网 快 ， 价格 可以 ， 就 早餐 不 好吃 。 整体 是 不错 的 。 適 合 出差 來 住 。 -2 本来 准备 住 两 晚 ， 第 2 天 一早 居然 停电 ， 且 无 通知 ， 只有 口头 道歉 。 总体来说 性价比 尚可 ， 房间 较 新 ， 还是 推荐 . -2 这个 酒店 去过 很多 次 了 ， 选择 的 主要原因 是 离 客户 最 便宜 相对 又 近 的 酒店 -2 挺好 的 汉庭 ， 前台 服务 很 热情 ， 卫生 很 整洁 ， 房间 安静 ， 水温 适中 ， 挺好 ！ -2 HowardJohnson 的 品质 ， 服务 相当 好 的 一 家 五星级 。 房间 不错 、 泳池 不错 、 楼层 安排 很 合理 。 还有 就是 地理位置 ， 简直 一 流 。 就 在 天一阁 、 月湖 旁边 ， 离 天一广场 也 不远 。 下次 来 宁波 还会 住 。 -2 酒店 很干净 ， 很安静 ， 很 温馨 ， 服务员 服务 好 ， 各方面 都 不错 * -2 挺好 的 ， 就是 没 窗户 ， 不过 对 得 起 这 价格 -``` - -- 双层序列的数据（`Sequence/tour_train_wdseg.nest`）如下，一共有4个样本。样本间用空行分开，代表不同的双层序列，序列数据和上面的完全一样。每个样本的子句数分别为2,3,2,3。 - -```text -2 酒店 有 很 舒适 的 床垫 子 ， 床上用品 也 应该 是 一人 一 换 ， 感觉 很 利落 对 卫生 很 放心 呀 。 -2 很 温馨 ， 也 挺 干净 的 * 地段 不错 ， 出来 就 有 全家 ， 离 地铁站 也 近 ， 交通 很方便 * 就是 都 不 给 刷牙 的 杯子 啊 ， 就 第一天 给 了 一次性杯子 * - -2 位置 方便 ， 强烈推荐 ， 十一 出去玩 的 时候 选 的 ， 对面 就是 华润万家 ， 周围 吃饭 的 也 不少 。 -2 交通便利 ， 吃 很 便利 ， 乾 浄 、 安静 ， 商务 房 有 电脑 、 上网 快 ， 价格 可以 ， 就 早餐 不 好吃 。 整体 是 不错 的 。 適 合 出差 來 住 。 -2 本来 准备 住 两 晚 ， 第 2 天 一早 居然 停电 ， 且 无 通知 ， 只有 口头 道歉 。 总体来说 性价比 尚可 ， 房间 较 新 ， 还是 推荐 . - -2 这个 酒店 去过 很多 次 了 ， 选择 的 主要原因 是 离 客户 最 便宜 相对 又 近 的 酒店 -2 挺好 的 汉庭 ， 前台 服务 很 热情 ， 卫生 很 整洁 ， 房间 安静 ， 水温 适中 ， 挺好 ！ - -2 HowardJohnson 的 品质 ， 服务 相当 好 的 一 家 五星级 。 房间 不错 、 泳池 不错 、 楼层 安排 很 合理 。 还有 就是 地理位置 ， 简直 一 流 。 就 在 天一阁 、 月湖 旁边 ， 离 天一广场 也 不远 。 下次 来 宁波 还会 住 。 -2 酒店 很干净 ， 很安静 ， 很 温馨 ， 服务员 服务 好 ， 各方面 都 不错 * -2 挺好 的 ， 就是 没 窗户 ， 不过 对 得 起 这 价格 -``` - -其次，我们看一下单双层序列的不同dataprovider（见`sequenceGen.py`）： - -- 单层序列的dataprovider如下： - - word_slot是integer_value_sequence类型，代表单层序列。 - - label是integer_value类型，代表一个向量。 - -```python -def hook(settings, dict_file, **kwargs): - settings.word_dict = dict_file - settings.input_types = [integer_value_sequence(len(settings.word_dict)), - integer_value(3)] - -@provider(init_hook=hook) -def process(settings, file_name): - with open(file_name, 'r') as fdata: - for line in fdata: - label, comment = line.strip().split('\t') - label = int(''.join(label.split())) - words = comment.split() - word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] - yield word_slot, label -``` - -- 双层序列的dataprovider如下： - - word_slot是integer_value_sub_sequence类型，代表双层序列。 - - label是integer_value_sequence类型，代表单层序列，即一个子句一个label。注意：也可以为integer_value类型，代表一个向量，即一个句子一个label。通常根据任务需求进行不同设置。 - - 关于dataprovider中input_types的详细用法，参见PyDataProvider2。 - -```python -def hook2(settings, dict_file, **kwargs): - settings.word_dict = dict_file - settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), - integer_value_sequence(3)] - -@provider(init_hook=hook2) -def process2(settings, file_name): - with open(file_name) as fdata: - label_list = [] - word_slot_list = [] - for line in fdata: - if (len(line)) > 1: - label,comment = line.strip().split('\t') - label = int(''.join(label.split())) - words = comment.split() - word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] - label_list.append(label) - word_slot_list.append(word_slot) - else: - yield word_slot_list, label_list - label_list = [] - word_slot_list = [] -``` - -### 模型中的配置 - -首先，我们看一下单层序列的配置（见`sequence_layer_group.conf`）。注意：batchsize=5表示一次过5句单层序列，因此2个batch就可以完成1个pass。 - -```python -settings(batch_size=5) - -data = data_layer(name="word", size=dict_dim) - -emb = embedding_layer(input=data, size=word_dim) - -# (lstm_input + lstm) is equal to lstmemory -with mixed_layer(size=hidden_dim*4) as lstm_input: - lstm_input += full_matrix_projection(input=emb) - -lstm = lstmemory_group(input=lstm_input, - size=hidden_dim, - act=TanhActivation(), - gate_act=SigmoidActivation(), - state_act=TanhActivation(), - lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) - -lstm_last = last_seq(input=lstm) - -with mixed_layer(size=label_dim, - act=SoftmaxActivation(), - bias_attr=True) as output: - output += full_matrix_projection(input=lstm_last) - -outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) - -``` -其次，我们看一下语义相同的双层序列配置（见`sequence_nest_layer_group.conf`），并对其详细分析： - -- batchsize=2表示一次过2句双层序列。但从上面的数据格式可知，2句双层序列和5句单层序列的数据完全一样。 -- data_layer和embedding_layer不关心数据是否是序列格式，因此两个配置在这两层上的输出是一样的。 -- lstmemory: - - 单层序列过了一个mixed_layer和lstmemory_group。 - - 双层序列在同样的mixed_layer和lstmemory_group外，直接加了一层group。由于这个外层group里面没有memory，表示subseq间不存在联系，即起到的作用仅仅是把双层seq拆成单层，因此双层序列过完lstmemory的输出和单层的一样。 -- last_seq： - - 单层序列直接取了最后一个元素 - - 双层序列首先（last_seq层）取了每个subseq的最后一个元素，将其拼接成一个新的单层序列；接着（expand_layer层）将其扩展成一个新的双层序列，其中第i个subseq中的所有向量均为输入的单层序列中的第i个向量；最后（average_layer层）取了每个subseq的平均值。 - - 分析得出：第一个last_seq后，每个subseq的最后一个元素就等于单层序列的最后一个元素，而expand_layer和average_layer后，依然保持每个subseq最后一个元素的值不变（这两层仅是为了展示它们的用法，实际中并不需要）。因此单双层序列的输出是一样旳。 - -```python -settings(batch_size=2) - -data = data_layer(name="word", size=dict_dim) - -emb_group = embedding_layer(input=data, size=word_dim) - -# (lstm_input + lstm) is equal to lstmemory -def lstm_group(lstm_group_input): - with mixed_layer(size=hidden_dim*4) as group_input: - group_input += full_matrix_projection(input=lstm_group_input) - - lstm_output = lstmemory_group(input=group_input, - name="lstm_group", - size=hidden_dim, - act=TanhActivation(), - gate_act=SigmoidActivation(), - state_act=TanhActivation(), - lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) - return lstm_output - -lstm_nest_group = recurrent_group(input=SubsequenceInput(emb_group), - step=lstm_group, - name="lstm_nest_group") -# hasSubseq ->(seqlastins) seq -lstm_last = last_seq(input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) - -# seq ->(expand) hasSubseq -lstm_expand = expand_layer(input=lstm_last, expand_as=emb_group, expand_level=ExpandLevel.FROM_SEQUENCE) - -# hasSubseq ->(average) seq -lstm_average = pooling_layer(input=lstm_expand, - pooling_type=AvgPooling(), - agg_level=AggregateLevel.EACH_SEQUENCE) - -with mixed_layer(size=label_dim, - act=SoftmaxActivation(), - bias_attr=True) as output: - output += full_matrix_projection(input=lstm_average) - -outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) -``` -## 示例2：双进双出，subseq间有memory - -配置：单层RNN（`sequence_rnn.conf`），双层RNN（`sequence_nest_rnn.conf`和`sequence_nest_rnn_readonly_memory.conf`），语义完全相同。 - -### 读取双层序列的方法 - -我们看一下单双层序列的不同数据组织形式和dataprovider（见`rnn_data_provider.py`） -```python -data = [ - [[[1, 3, 2], [4, 5, 2]], 0], - [[[0, 2], [2, 5], [0, 1, 2]], 1], -] - -@provider(input_types=[integer_value_sub_sequence(10), - integer_value(3)]) -def process_subseq(settings, file_name): - for d in data: - yield d - -@provider(input_types=[integer_value_sequence(10), - integer_value(3)]) -def process_seq(settings, file_name): - for d in data: - seq = [] -``` -- 单层序列：有两句，分别为[1,3,2,4,5,2]和[0,2,2,5,0,1,2]。 -- 双层序列：有两句，分别为[[1,3,2],[4,5,2]]（2个子句）和[[0,2],[2,5],[0,1,2]]（3个子句）。 -- 单双层序列的label都分别是0和1 - -### 模型中的配置 - -我们选取单双层序列配置中的不同部分，来对比分析两者语义相同的原因。 - -- 单层序列：过了一个很简单的recurrent_group。每一个时间步，当前的输入y和上一个时间步的输出rnn_state做了一个全链接。 - -```python -def step(y): - mem = memory(name="rnn_state", size=hidden_dim) - return fc_layer(input=[y, mem], - size=hidden_dim, - act=TanhActivation(), - bias_attr=True, - name="rnn_state") - -out = recurrent_group(step=step, input=emb) -``` -- 双层序列，外层memory是一个元素： - - 内层inner_step的recurrent_group和单层序列的几乎一样。除了boot_layer=outer_mem，表示将外层的outer_mem作为内层memory的初始状态。外层outer_step中，outer_mem是一个子句的最后一个向量，即整个双层group是将前一个子句的最后一个向量，作为下一个子句memory的初始状态。 - - 从输入数据上看，单双层序列的句子是一样的，只是双层序列将其又做了子序列划分。因此双层序列的配置中，必须将前一个子句的最后一个元素，作为boot_layer传给下一个子句的memory，才能保证和单层序列的配置中“每一个时间步都用了上一个时间步的输出结果”一致。 - -```python -def outer_step(x): - outer_mem = memory(name="outer_rnn_state", size=hidden_dim) - def inner_step(y): - inner_mem = memory(name="inner_rnn_state", - size=hidden_dim, - boot_layer=outer_mem) - return fc_layer(input=[y, inner_mem], - size=hidden_dim, - act=TanhActivation(), - bias_attr=True, - name="inner_rnn_state") - - inner_rnn_output = recurrent_group( - step=inner_step, - input=x) - last = last_seq(input=inner_rnn_output, name="outer_rnn_state") - - return inner_rnn_output - -out = recurrent_group(step=outer_step, input=SubsequenceInput(emb)) -``` -- 双层序列，外层memory是单层序列： - - 由于外层每个时间步返回的是一个子句，这些子句的长度往往不等长。因此当外层有is_seq=True的memory时，内层是**无法直接使用**它的，即内层memory的boot_layer不能链接外层的这个memory。 - - 如果内层memory想**间接使用**这个外层memory，只能通过`pooling_layer`、`last_seq`或`first_seq`这三个layer将它先变成一个元素。但这种情况下，外层memory必须有boot_layer，否则在第0个时间步时，由于外层memory没有任何seq信息，因此上述三个layer的前向会报出“**Check failed: input.sequenceStartPositions**”的错误。 - -## 示例3：双进双出，输入不等长 - -**输入不等长**是指recurrent_group的多个输入在各时刻的长度可以不相等, 但需要指定一个和输出长度一致的input，用targetInlink表示。参考配置：单层RNN（`sequence_rnn_multi_unequalength_inputs.conf`），双层RNN（`sequence_nest_rnn_multi_unequalength_inputs.conf`） - -### 读取双层序列的方法 - -我们看一下单双层序列的数据组织形式和dataprovider（见`rnn_data_provider.py`） -```python -data2 = [ - [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]] ,0], - [[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]], 1], -] - -@provider(input_types=[integer_value_sub_sequence(10), - integer_value_sub_sequence(10), - integer_value(2)], - should_shuffle=False) -def process_unequalength_subseq(settings, file_name): #双层RNN的dataprovider - for d in data2: - yield d - - -@provider(input_types=[integer_value_sequence(10), - integer_value_sequence(10), - integer_value(2)], - should_shuffle=False) -def process_unequalength_seq(settings, file_name): #单层RNN的dataprovider - for d in data2: - words1=reduce(lambda x,y: x+y, d[0]) - words2=reduce(lambda x,y: x+y, d[1]) - yield words1, words2, d[2] -``` - -data2 中有两个样本，每个样本有两个特征, 记fea1, fea2。 - -- 单层序列：两个样本分别为[[1, 2, 4, 5, 2], [5, 4, 1, 3, 1]] 和 [[0, 2, 2, 5, 0, 1, 2], [1, 5, 4, 2, 3, 6, 1]] -- 双层序列：两个样本分别为 - - **样本1**：[[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]]]。fea1和fea2都分别有2个子句，fea1=[[1, 2], [4, 5, 2]], fea2=[[5, 4, 1], [3, 1]] - - **样本2**：[[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]]]。fea1和fea2都分别有3个子句， fea1=[[0, 2], [2, 5], [0, 1, 2]], fea2=[[1, 5], [4], [2, 3, 6, 1]]。
- - **注意**：每个样本中，各特征的子句数目需要相等。这里说的“双进双出，输入不等长”是指fea1在i时刻的输入的长度可以不等于fea2在i时刻的输入的长度。如对于第1个样本，时刻i=2, fea1[2]=[4, 5, 2]，fea2[2]=[3, 1]，3≠2。 -- 单双层序列中，两个样本的label都分别是0和1 - -### 模型中的配置 - -单层RNN（`sequence_rnn_multi_unequalength_inputs.conf`）和双层RNN（`sequence_nest_rnn_multi_unequalength_inputs.conf`）两个模型配置达到的效果完全一样，区别只在于输入为单层还是双层序列，现在我们来看它们内部分别是如何实现的。 - -- 单层序列： - - 过了一个简单的recurrent_group。每一个时间步，当前的输入y和上一个时间步的输出rnn_state做了一个全连接，功能与示例2中`sequence_rnn.conf`的`step`函数完全相同。这里，两个输入x1,x2分别通过calrnn返回最后时刻的状态。结果得到的encoder1_rep和encoder2_rep分别是单层序列，最后取encoder1_rep的最后一个时刻和encoder2_rep的所有时刻分别相加得到context。 - - 注意到这里recurrent_group输入的每个样本中，fea1和fea2的长度都分别相等，这并非偶然，而是因为recurrent_group要求输入为单层序列时，所有输入的长度都必须相等。 - -```python -def step(x1, x2): - def calrnn(y): - mem = memory(name = 'rnn_state_' + y.name, size = hidden_dim) - out = fc_layer(input = [y, mem], - size = hidden_dim, - act = TanhActivation(), - bias_attr = True, - name = 'rnn_state_' + y.name) - return out - - encoder1 = calrnn(x1) - encoder2 = calrnn(x2) - return [encoder1, encoder2] - -encoder1_rep, encoder2_rep = recurrent_group( - name="stepout", - step=step, - input=[emb1, emb2]) - -encoder1_last = last_seq(input = encoder1_rep) -encoder1_expandlast = expand_layer(input = encoder1_last, - expand_as = encoder2_rep) -context = mixed_layer(input = [identity_projection(encoder1_expandlast), - identity_projection(encoder2_rep)], - size = hidden_dim) -``` -- 双层序列： - - 双层RNN中，对输入的两个特征分别求时序上的连续全连接(`inner_step1`和`inner_step2`分别处理fea1和fea2)，其功能与示例2中`sequence_nest_rnn.conf`的`outer_step`函数完全相同。不同之处是，此时输入`[SubsequenceInput(emb1), SubsequenceInput(emb2)]`在各时刻并不等长。 - - 函数`outer_step`中可以分别处理这两个特征，但我们需要用targetInlink指定recurrent_group的输出的格式（各子句长度）只能和其中一个保持一致，如这里选择了和emb2的长度一致。 - - 最后，依然是取encoder1_rep的最后一个时刻和encoder2_rep的所有时刻分别相加得到context。 - -```python -def outer_step(x1, x2): - outer_mem1 = memory(name = "outer_rnn_state1", size = hidden_dim) - outer_mem2 = memory(name = "outer_rnn_state2", size = hidden_dim) - def inner_step1(y): - inner_mem = memory(name = 'inner_rnn_state_' + y.name, - size = hidden_dim, - boot_layer = outer_mem1) - out = fc_layer(input = [y, inner_mem], - size = hidden_dim, - act = TanhActivation(), - bias_attr = True, - name = 'inner_rnn_state_' + y.name) - return out - - def inner_step2(y): - inner_mem = memory(name = 'inner_rnn_state_' + y.name, - size = hidden_dim, - boot_layer = outer_mem2) - out = fc_layer(input = [y, inner_mem], - size = hidden_dim, - act = TanhActivation(), - bias_attr = True, - name = 'inner_rnn_state_' + y.name) - return out - - encoder1 = recurrent_group( - step = inner_step1, - name = 'inner1', - input = x1) - - encoder2 = recurrent_group( - step = inner_step2, - name = 'inner2', - input = x2) - - sentence_last_state1 = last_seq(input = encoder1, name = 'outer_rnn_state1') - sentence_last_state2_ = last_seq(input = encoder2, name = 'outer_rnn_state2') - - encoder1_expand = expand_layer(input = sentence_last_state1, - expand_as = encoder2) - - return [encoder1_expand, encoder2] - -encoder1_rep, encoder2_rep = recurrent_group( - name="outer", - step=outer_step, - input=[SubsequenceInput(emb1), SubsequenceInput(emb2)], - targetInlink=emb2) - -encoder1_last = last_seq(input = encoder1_rep) -encoder1_expandlast = expand_layer(input = encoder1_last, - expand_as = encoder2_rep) -context = mixed_layer(input = [identity_projection(encoder1_expandlast), - identity_projection(encoder2_rep)], - size = hidden_dim) -``` - -## 示例4：beam_search的生成 - -TBD \ No newline at end of file +# 双层RNN配置与示例 + +我们在`paddle/gserver/tests/test_RecurrentGradientMachine`单测中，通过多组语义相同的单双层RNN配置，讲解如何使用双层RNN。 + +## 示例1：双进双出，subseq间无memory + +配置：单层RNN（`sequence_layer_group`）和双层RNN（`sequence_nest_layer_group`），语义完全相同。 + +### 读取双层序列的方法 + +首先，我们看一下单双层序列的不同数据组织形式（您也可以采用别的组织形式）： + +- 单层序列的数据（`Sequence/tour_train_wdseg`）如下，一共有10个样本。每个样本由两部分组成，一个label（此处都为2）和一个已经分词后的句子。 + +```text +2 酒店 有 很 舒适 的 床垫 子 ， 床上用品 也 应该 是 一人 一 换 ， 感觉 很 利落 对 卫生 很 放心 呀 。 +2 很 温馨 ， 也 挺 干净 的 * 地段 不错 ， 出来 就 有 全家 ， 离 地铁站 也 近 ， 交通 很方便 * 就是 都 不 给 刷牙 的 杯子 啊 ， 就 第一天 给 了 一次性杯子 * +2 位置 方便 ， 强烈推荐 ， 十一 出去玩 的 时候 选 的 ， 对面 就是 华润万家 ， 周围 吃饭 的 也 不少 。 +2 交通便利 ， 吃 很 便利 ， 乾 浄 、 安静 ， 商务 房 有 电脑 、 上网 快 ， 价格 可以 ， 就 早餐 不 好吃 。 整体 是 不错 的 。 適 合 出差 來 住 。 +2 本来 准备 住 两 晚 ， 第 2 天 一早 居然 停电 ， 且 无 通知 ， 只有 口头 道歉 。 总体来说 性价比 尚可 ， 房间 较 新 ， 还是 推荐 . +2 这个 酒店 去过 很多 次 了 ， 选择 的 主要原因 是 离 客户 最 便宜 相对 又 近 的 酒店 +2 挺好 的 汉庭 ， 前台 服务 很 热情 ， 卫生 很 整洁 ， 房间 安静 ， 水温 适中 ， 挺好 ！ +2 HowardJohnson 的 品质 ， 服务 相当 好 的 一 家 五星级 。 房间 不错 、 泳池 不错 、 楼层 安排 很 合理 。 还有 就是 地理位置 ， 简直 一 流 。 就 在 天一阁 、 月湖 旁边 ， 离 天一广场 也 不远 。 下次 来 宁波 还会 住 。 +2 酒店 很干净 ， 很安静 ， 很 温馨 ， 服务员 服务 好 ， 各方面 都 不错 * +2 挺好 的 ， 就是 没 窗户 ， 不过 对 得 起 这 价格 +``` + +- 双层序列的数据（`Sequence/tour_train_wdseg.nest`）如下，一共有4个样本。样本间用空行分开，代表不同的双层序列，序列数据和上面的完全一样。每个样本的子句数分别为2,3,2,3。 + +```text +2 酒店 有 很 舒适 的 床垫 子 ， 床上用品 也 应该 是 一人 一 换 ， 感觉 很 利落 对 卫生 很 放心 呀 。 +2 很 温馨 ， 也 挺 干净 的 * 地段 不错 ， 出来 就 有 全家 ， 离 地铁站 也 近 ， 交通 很方便 * 就是 都 不 给 刷牙 的 杯子 啊 ， 就 第一天 给 了 一次性杯子 * + +2 位置 方便 ， 强烈推荐 ， 十一 出去玩 的 时候 选 的 ， 对面 就是 华润万家 ， 周围 吃饭 的 也 不少 。 +2 交通便利 ， 吃 很 便利 ， 乾 浄 、 安静 ， 商务 房 有 电脑 、 上网 快 ， 价格 可以 ， 就 早餐 不 好吃 。 整体 是 不错 的 。 適 合 出差 來 住 。 +2 本来 准备 住 两 晚 ， 第 2 天 一早 居然 停电 ， 且 无 通知 ， 只有 口头 道歉 。 总体来说 性价比 尚可 ， 房间 较 新 ， 还是 推荐 . + +2 这个 酒店 去过 很多 次 了 ， 选择 的 主要原因 是 离 客户 最 便宜 相对 又 近 的 酒店 +2 挺好 的 汉庭 ， 前台 服务 很 热情 ， 卫生 很 整洁 ， 房间 安静 ， 水温 适中 ， 挺好 ！ + +2 HowardJohnson 的 品质 ， 服务 相当 好 的 一 家 五星级 。 房间 不错 、 泳池 不错 、 楼层 安排 很 合理 。 还有 就是 地理位置 ， 简直 一 流 。 就 在 天一阁 、 月湖 旁边 ， 离 天一广场 也 不远 。 下次 来 宁波 还会 住 。 +2 酒店 很干净 ， 很安静 ， 很 温馨 ， 服务员 服务 好 ， 各方面 都 不错 * +2 挺好 的 ， 就是 没 窗户 ， 不过 对 得 起 这 价格 +``` + +其次，我们看一下单双层序列的不同dataprovider（见`sequenceGen.py`）： + +- 单层序列的dataprovider如下： + - word_slot是integer_value_sequence类型，代表单层序列。 + - label是integer_value类型，代表一个向量。 + +```python +def hook(settings, dict_file, **kwargs): + settings.word_dict = dict_file + settings.input_types = [integer_value_sequence(len(settings.word_dict)), + integer_value(3)] + +@provider(init_hook=hook) +def process(settings, file_name): + with open(file_name, 'r') as fdata: + for line in fdata: + label, comment = line.strip().split('\t') + label = int(''.join(label.split())) + words = comment.split() + word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] + yield word_slot, label +``` + +- 双层序列的dataprovider如下： + - word_slot是integer_value_sub_sequence类型，代表双层序列。 + - label是integer_value_sequence类型，代表单层序列，即一个子句一个label。注意：也可以为integer_value类型，代表一个向量，即一个句子一个label。通常根据任务需求进行不同设置。 + - 关于dataprovider中input_types的详细用法，参见PyDataProvider2。 + +```python +def hook2(settings, dict_file, **kwargs): + settings.word_dict = dict_file + settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), + integer_value_sequence(3)] + +@provider(init_hook=hook2) +def process2(settings, file_name): + with open(file_name) as fdata: + label_list = [] + word_slot_list = [] + for line in fdata: + if (len(line)) > 1: + label,comment = line.strip().split('\t') + label = int(''.join(label.split())) + words = comment.split() + word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] + label_list.append(label) + word_slot_list.append(word_slot) + else: + yield word_slot_list, label_list + label_list = [] + word_slot_list = [] +``` + +### 模型中的配置 + +首先，我们看一下单层序列的配置（见`sequence_layer_group.conf`）。注意：batchsize=5表示一次过5句单层序列，因此2个batch就可以完成1个pass。 + +```python +settings(batch_size=5) + +data = data_layer(name="word", size=dict_dim) + +emb = embedding_layer(input=data, size=word_dim) + +# (lstm_input + lstm) is equal to lstmemory +with mixed_layer(size=hidden_dim*4) as lstm_input: + lstm_input += full_matrix_projection(input=emb) + +lstm = lstmemory_group(input=lstm_input, + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + +lstm_last = last_seq(input=lstm) + +with mixed_layer(size=label_dim, + act=SoftmaxActivation(), + bias_attr=True) as output: + output += full_matrix_projection(input=lstm_last) + +outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) + +``` +其次，我们看一下语义相同的双层序列配置（见`sequence_nest_layer_group.conf`），并对其详细分析： + +- batchsize=2表示一次过2句双层序列。但从上面的数据格式可知，2句双层序列和5句单层序列的数据完全一样。 +- data_layer和embedding_layer不关心数据是否是序列格式，因此两个配置在这两层上的输出是一样的。 +- lstmemory: + - 单层序列过了一个mixed_layer和lstmemory_group。 + - 双层序列在同样的mixed_layer和lstmemory_group外，直接加了一层group。由于这个外层group里面没有memory，表示subseq间不存在联系，即起到的作用仅仅是把双层seq拆成单层，因此双层序列过完lstmemory的输出和单层的一样。 +- last_seq： + - 单层序列直接取了最后一个元素 + - 双层序列首先（last_seq层）取了每个subseq的最后一个元素，将其拼接成一个新的单层序列；接着（expand_layer层）将其扩展成一个新的双层序列，其中第i个subseq中的所有向量均为输入的单层序列中的第i个向量；最后（average_layer层）取了每个subseq的平均值。 + - 分析得出：第一个last_seq后，每个subseq的最后一个元素就等于单层序列的最后一个元素，而expand_layer和average_layer后，依然保持每个subseq最后一个元素的值不变（这两层仅是为了展示它们的用法，实际中并不需要）。因此单双层序列的输出是一样旳。 + +```python +settings(batch_size=2) + +data = data_layer(name="word", size=dict_dim) + +emb_group = embedding_layer(input=data, size=word_dim) + +# (lstm_input + lstm) is equal to lstmemory +def lstm_group(lstm_group_input): + with mixed_layer(size=hidden_dim*4) as group_input: + group_input += full_matrix_projection(input=lstm_group_input) + + lstm_output = lstmemory_group(input=group_input, + name="lstm_group", + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + return lstm_output + +lstm_nest_group = recurrent_group(input=SubsequenceInput(emb_group), + step=lstm_group, + name="lstm_nest_group") +# hasSubseq ->(seqlastins) seq +lstm_last = last_seq(input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) + +# seq ->(expand) hasSubseq +lstm_expand = expand_layer(input=lstm_last, expand_as=emb_group, expand_level=ExpandLevel.FROM_SEQUENCE) + +# hasSubseq ->(average) seq +lstm_average = pooling_layer(input=lstm_expand, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) + +with mixed_layer(size=label_dim, + act=SoftmaxActivation(), + bias_attr=True) as output: + output += full_matrix_projection(input=lstm_average) + +outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +``` +## 示例2：双进双出，subseq间有memory + +配置：单层RNN（`sequence_rnn.conf`），双层RNN（`sequence_nest_rnn.conf`和`sequence_nest_rnn_readonly_memory.conf`），语义完全相同。 + +### 读取双层序列的方法 + +我们看一下单双层序列的不同数据组织形式和dataprovider（见`rnn_data_provider.py`） +```python +data = [ + [[[1, 3, 2], [4, 5, 2]], 0], + [[[0, 2], [2, 5], [0, 1, 2]], 1], +] + +@provider(input_types=[integer_value_sub_sequence(10), + integer_value(3)]) +def process_subseq(settings, file_name): + for d in data: + yield d + +@provider(input_types=[integer_value_sequence(10), + integer_value(3)]) +def process_seq(settings, file_name): + for d in data: + seq = [] +``` +- 单层序列：有两句，分别为[1,3,2,4,5,2]和[0,2,2,5,0,1,2]。 +- 双层序列：有两句，分别为[[1,3,2],[4,5,2]]（2个子句）和[[0,2],[2,5],[0,1,2]]（3个子句）。 +- 单双层序列的label都分别是0和1 + +### 模型中的配置 + +我们选取单双层序列配置中的不同部分，来对比分析两者语义相同的原因。 + +- 单层序列：过了一个很简单的recurrent_group。每一个时间步，当前的输入y和上一个时间步的输出rnn_state做了一个全链接。 + +```python +def step(y): + mem = memory(name="rnn_state", size=hidden_dim) + return fc_layer(input=[y, mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="rnn_state") + +out = recurrent_group(step=step, input=emb) +``` +- 双层序列，外层memory是一个元素： + - 内层inner_step的recurrent_group和单层序列的几乎一样。除了boot_layer=outer_mem，表示将外层的outer_mem作为内层memory的初始状态。外层outer_step中，outer_mem是一个子句的最后一个向量，即整个双层group是将前一个子句的最后一个向量，作为下一个子句memory的初始状态。 + - 从输入数据上看，单双层序列的句子是一样的，只是双层序列将其又做了子序列划分。因此双层序列的配置中，必须将前一个子句的最后一个元素，作为boot_layer传给下一个子句的memory，才能保证和单层序列的配置中“每一个时间步都用了上一个时间步的输出结果”一致。 + +```python +def outer_step(x): + outer_mem = memory(name="outer_rnn_state", size=hidden_dim) + def inner_step(y): + inner_mem = memory(name="inner_rnn_state", + size=hidden_dim, + boot_layer=outer_mem) + return fc_layer(input=[y, inner_mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="inner_rnn_state") + + inner_rnn_output = recurrent_group( + step=inner_step, + input=x) + last = last_seq(input=inner_rnn_output, name="outer_rnn_state") + + return inner_rnn_output + +out = recurrent_group(step=outer_step, input=SubsequenceInput(emb)) +``` +- 双层序列，外层memory是单层序列： + - 由于外层每个时间步返回的是一个子句，这些子句的长度往往不等长。因此当外层有is_seq=True的memory时，内层是**无法直接使用**它的，即内层memory的boot_layer不能链接外层的这个memory。 + - 如果内层memory想**间接使用**这个外层memory，只能通过`pooling_layer`、`last_seq`或`first_seq`这三个layer将它先变成一个元素。但这种情况下，外层memory必须有boot_layer，否则在第0个时间步时，由于外层memory没有任何seq信息，因此上述三个layer的前向会报出“**Check failed: input.sequenceStartPositions**”的错误。 + +## 示例3：双进双出，输入不等长 + +**输入不等长**是指recurrent_group的多个输入在各时刻的长度可以不相等, 但需要指定一个和输出长度一致的input，用targetInlink表示。参考配置：单层RNN（`sequence_rnn_multi_unequalength_inputs.conf`），双层RNN（`sequence_nest_rnn_multi_unequalength_inputs.conf`） + +### 读取双层序列的方法 + +我们看一下单双层序列的数据组织形式和dataprovider（见`rnn_data_provider.py`） +```python +data2 = [ + [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]] ,0], + [[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]], 1], +] + +@provider(input_types=[integer_value_sub_sequence(10), + integer_value_sub_sequence(10), + integer_value(2)], + should_shuffle=False) +def process_unequalength_subseq(settings, file_name): #双层RNN的dataprovider + for d in data2: + yield d + + +@provider(input_types=[integer_value_sequence(10), + integer_value_sequence(10), + integer_value(2)], + should_shuffle=False) +def process_unequalength_seq(settings, file_name): #单层RNN的dataprovider + for d in data2: + words1=reduce(lambda x,y: x+y, d[0]) + words2=reduce(lambda x,y: x+y, d[1]) + yield words1, words2, d[2] +``` + +data2 中有两个样本，每个样本有两个特征, 记fea1, fea2。 + +- 单层序列：两个样本分别为[[1, 2, 4, 5, 2], [5, 4, 1, 3, 1]] 和 [[0, 2, 2, 5, 0, 1, 2], [1, 5, 4, 2, 3, 6, 1]] +- 双层序列：两个样本分别为 + - **样本1**：[[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]]]。fea1和fea2都分别有2个子句，fea1=[[1, 2], [4, 5, 2]], fea2=[[5, 4, 1], [3, 1]] + - **样本2**：[[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]]]。fea1和fea2都分别有3个子句， fea1=[[0, 2], [2, 5], [0, 1, 2]], fea2=[[1, 5], [4], [2, 3, 6, 1]]。
+ - **注意**：每个样本中，各特征的子句数目需要相等。这里说的“双进双出，输入不等长”是指fea1在i时刻的输入的长度可以不等于fea2在i时刻的输入的长度。如对于第1个样本，时刻i=2, fea1[2]=[4, 5, 2]，fea2[2]=[3, 1]，3≠2。 +- 单双层序列中，两个样本的label都分别是0和1 + +### 模型中的配置 + +单层RNN（`sequence_rnn_multi_unequalength_inputs.conf`）和双层RNN（`sequence_nest_rnn_multi_unequalength_inputs.conf`）两个模型配置达到的效果完全一样，区别只在于输入为单层还是双层序列，现在我们来看它们内部分别是如何实现的。 + +- 单层序列： + - 过了一个简单的recurrent_group。每一个时间步，当前的输入y和上一个时间步的输出rnn_state做了一个全连接，功能与示例2中`sequence_rnn.conf`的`step`函数完全相同。这里，两个输入x1,x2分别通过calrnn返回最后时刻的状态。结果得到的encoder1_rep和encoder2_rep分别是单层序列，最后取encoder1_rep的最后一个时刻和encoder2_rep的所有时刻分别相加得到context。 + - 注意到这里recurrent_group输入的每个样本中，fea1和fea2的长度都分别相等，这并非偶然，而是因为recurrent_group要求输入为单层序列时，所有输入的长度都必须相等。 + +```python +def step(x1, x2): + def calrnn(y): + mem = memory(name = 'rnn_state_' + y.name, size = hidden_dim) + out = fc_layer(input = [y, mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'rnn_state_' + y.name) + return out + + encoder1 = calrnn(x1) + encoder2 = calrnn(x2) + return [encoder1, encoder2] + +encoder1_rep, encoder2_rep = recurrent_group( + name="stepout", + step=step, + input=[emb1, emb2]) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) +``` +- 双层序列： + - 双层RNN中，对输入的两个特征分别求时序上的连续全连接(`inner_step1`和`inner_step2`分别处理fea1和fea2)，其功能与示例2中`sequence_nest_rnn.conf`的`outer_step`函数完全相同。不同之处是，此时输入`[SubsequenceInput(emb1), SubsequenceInput(emb2)]`在各时刻并不等长。 + - 函数`outer_step`中可以分别处理这两个特征，但我们需要用targetInlink指定recurrent_group的输出的格式（各子句长度）只能和其中一个保持一致，如这里选择了和emb2的长度一致。 + - 最后，依然是取encoder1_rep的最后一个时刻和encoder2_rep的所有时刻分别相加得到context。 + +```python +def outer_step(x1, x2): + outer_mem1 = memory(name = "outer_rnn_state1", size = hidden_dim) + outer_mem2 = memory(name = "outer_rnn_state2", size = hidden_dim) + def inner_step1(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem1) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + def inner_step2(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem2) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + encoder1 = recurrent_group( + step = inner_step1, + name = 'inner1', + input = x1) + + encoder2 = recurrent_group( + step = inner_step2, + name = 'inner2', + input = x2) + + sentence_last_state1 = last_seq(input = encoder1, name = 'outer_rnn_state1') + sentence_last_state2_ = last_seq(input = encoder2, name = 'outer_rnn_state2') + + encoder1_expand = expand_layer(input = sentence_last_state1, + expand_as = encoder2) + + return [encoder1_expand, encoder2] + +encoder1_rep, encoder2_rep = recurrent_group( + name="outer", + step=outer_step, + input=[SubsequenceInput(emb1), SubsequenceInput(emb2)], + targetInlink=emb2) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) +``` + +## 示例4：beam_search的生成 + +TBD diff --git a/doc_cn/algorithm/rnn/rnn-tutorial.md b/doc_cn/algorithm/rnn/rnn-tutorial.md index 7a553054c80392946ba5b16cc31bcaea18cfc977..9e488b0d51956e86f9fb76f450fdb438f596e239 100644 --- a/doc_cn/algorithm/rnn/rnn-tutorial.md +++ b/doc_cn/algorithm/rnn/rnn-tutorial.md @@ -93,4 +93,4 @@ memory只能在`recurrent_group`中定义和使用。memory不能独立存在， 使用`beam_search`需要遵循以下约定： - 单层RNN：从一个word生成下一个word。 -- 双层RNN：即把单层RNN生成后的subseq给拼接成一个新的双层seq。从语义上看，也不存在一个subseq直接生成下一个subseq的情况。 \ No newline at end of file +- 双层RNN：即把单层RNN生成后的subseq给拼接成一个新的双层seq。从语义上看，也不存在一个subseq直接生成下一个subseq的情况。 diff --git a/doc_cn/build_and_install/install/paddle_version.txt b/doc_cn/build_and_install/install/paddle_version.txt index 7b2bfd2b1b3a9850665d118e424fd0cf6c24a062..a80873303fd0d05d963482629000d76260185ef6 100644 --- a/doc_cn/build_and_install/install/paddle_version.txt +++ b/doc_cn/build_and_install/install/paddle_version.txt @@ -8,4 +8,4 @@ PaddlePaddle 0.8.0b1, compiled with with_gflags: ON with_metric_learning: with_timer: OFF - with_predict_sdk: \ No newline at end of file + with_predict_sdk: diff --git a/doc_cn/concepts/trainer_config.py b/doc_cn/concepts/trainer_config.py index 8d8c79fb39e0c0ddf13aee5d41297506d3404362..3eccbd7bc11f4865130286de718d1be74e4d1722 100644 --- a/doc_cn/concepts/trainer_config.py +++ b/doc_cn/concepts/trainer_config.py @@ -1,23 +1,29 @@ from paddle.trainer_config_helpers import * -define_py_data_sources2(train_list='train.list', - test_list='test.list', - module='provider', - obj='process') +define_py_data_sources2( + train_list='train.list', + test_list='test.list', + module='provider', + obj='process') settings( batch_size=128, learning_rate=1e-3, learning_method=AdamOptimizer(), - regularization=L2Regularization(0.5) -) + regularization=L2Regularization(0.5)) img = data_layer(name='pixel', size=28 * 28) -hidden1 = simple_img_conv_pool(input=img, filter_size=3, num_filters=32, pool_size=3, - num_channel=1) +hidden1 = simple_img_conv_pool( + input=img, filter_size=3, num_filters=32, pool_size=3, num_channel=1) -hidden2 = fc_layer(input=hidden1, size=200, act=TanhActivation(), - layer_attr=ExtraAttr(drop_rate=0.5)) +hidden2 = fc_layer( + input=hidden1, + size=200, + act=TanhActivation(), + layer_attr=ExtraAttr(drop_rate=0.5)) predict = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) -outputs(classification_cost(input=predict, label=data_layer(name='label', size=10))) +outputs( + classification_cost( + input=predict, label=data_layer( + name='label', size=10))) diff --git a/doc_cn/demo/index.rst b/doc_cn/demo/index.rst index 71f54bc18fbb5b1b8cdd0e6cbee2ee028c0af218..e15e839f93d4ac0d455e49fd8b1cde8bf60a29ac 100644 --- a/doc_cn/demo/index.rst +++ b/doc_cn/demo/index.rst @@ -9,7 +9,7 @@ 自然语言处理 '''''''''''' -* `情感分析 <../../doc/demo/sentiment_analysis/index.html>`_ +* `情感分析 `_ * `文本生成 <../../doc/demo/text_generation/index.html>`_ * `词性标注 <../../doc/demo/semantic_role_labeling/index.html>`_ diff --git a/doc_cn/demo/sentiment_analysis/index.rst b/doc_cn/demo/sentiment_analysis/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..82400b2459ebcaf89ff5e884edfe721b9ec01d7f --- /dev/null +++ b/doc_cn/demo/sentiment_analysis/index.rst @@ -0,0 +1,8 @@ +情感分析教程 +=========================== + +.. toctree:: + :maxdepth: 3 + :glob: + + Training Locally \ No newline at end of file diff --git a/doc_cn/demo/sentiment_analysis/sentiment_analysis.md b/doc_cn/demo/sentiment_analysis/sentiment_analysis.md new file mode 100644 index 0000000000000000000000000000000000000000..b70f2d59675615c26b29932cdf99d728bb206148 --- /dev/null +++ b/doc_cn/demo/sentiment_analysis/sentiment_analysis.md @@ -0,0 +1,324 @@ +# 情感分析教程 + +情感分析有许多应用场景。 一个基本的应用场景是区分给定文本的褒贬两极性，给定的文本可以是一个文档、句子、或者是一个小的文本片段。 一个简单的例子如：把用户在购物网站、旅游网站、团购网站（亚马逊、天猫、淘宝等）上发表的评论分成正面评论和负面评论两类。 + +情感分析也常用于基于大量评论和个人博客来监控社会媒体。 例如，研究人员分析了几个关于消费者信心和政治观点的调查，结果发现它们与同时期的Twitter消息中的情绪词频率相关 [1]。 另一个例子是通过分析每日Twitter博客的文本内容来预测股票变动 [2]。 + +另一方面，抓取产品的用户评论并分析他们的情感，有助于理解用户对不同公司，不同产品，甚至不同竞争对手产品的偏好。 + +本教程将指导您完成长期短期记忆（LSTM）网络的训练过程，以分类来自[大型电影评论数据集](http://ai.stanford.edu/~amaas/data/sentiment/)（有时称为[互联网电影数据库 (IMDB)](http://ai.stanford.edu/~amaas/papers/wvSent_acl2011.pdf)）的句子的情感 。 此数据集包含电影评论及其相关联的类别标签，即正面和负面。 + +## 数椐准备 + +### IMDB 数椐介绍 + +训练模型之前, 我们需要预处理数椐并构建一个字典。 首先, 你可以使用下面的脚本下载 IMDB 数椐集和[Moses](http://www.statmt.org/moses/)工具, 这是一个基于统计的机器翻译系统. 我们提供了一个数据预处理脚本，它不仅能够处理IMDB数据，还能处理其他用户自定义的数据。 为了使用提前编写的脚本，需要将标记的训练和测试样本移动到另一个路径，这已经在`get_imdb.sh`中完成。 + +``` +cd demo/sentiment/data +./get_imdb.sh +``` +如果数椐获取成功，你将在目录```./demo/sentiment/data```中看到下面的文件： + +``` +aclImdb get_imdb.sh imdb mosesdecoder-master +``` + +* aclImdb: 从外部网站上下载的原始数椐集。 +* imdb: 仅包含训练和测试数椐集。 +* mosesdecoder-master: Moses 工具。 + +IMDB数据集包含25,000个已标注过的高极性电影评论用于训练，25,000个用于测试。负面的评论的得分小于等于4,正面的评论的得大于等于7，总评分10分。 运行完脚本 `./get_imdb.sh`后, 我们可以看到在目录 `aclImdb`中的数椐集的结构如下： + +``` +imdbEr.txt imdb.vocab README test train +``` +* train: 训练数椐集。 +* test : 测试数椐集。 +* imdb.vocab: 字典文件。 +* imdbEr.txt: 字典imdb.vocab中每个切分单词的预期评级。 +* README: 数椐说明文档。 + +测试集和训练集目录包含下面的文件: + +``` +labeledBow.feat neg pos unsup unsupBow.feat urls_neg.txt urls_pos.txt urls_unsup.txt +``` + +* pos: 正面评价样本，包含12,500个txt文件，每个文件是一个电影评论。 +* neg: 负面评价样本，包含12,500个txt文件，每个文件是一个电影评论。 +* unsup: 未标记的评价样本，包含50,000个txt文件。 +* urls_xx.txt: 每个评论的网址。 +* xxBow.feat: 用于统计词频的Bow模型特征。 + +### IMDB 数椐准备 + +在这个例子中，我们只使用已经标注过的训练集和测试集，且默认在训练集上构建字典，而不使用IMDB数椐集中的imdb.vocab做为字典。训练集已经做了随机打乱排序而测试集没有。 Moses 工具中的脚本`tokenizer.perl` 用于切分单单词和标点符号。执行下面的命令就可以预处理数椐。 + +``` +cd demo/sentiment/ +./preprocess.sh +``` +preprocess.sh: + +``` +data_dir="./data/imdb" +python preprocess.py -i data_dir +``` + +* data_dir: 输入数椐所在目录。 +* preprocess.py: 预处理脚本。 + +运行成功后目录`demo/sentiment/data/pre-imdb` 结构如下: + +``` +dict.txt labels.list test.list test_part_000 train.list train_part_000 +``` +* test\_part\_000 and train\_part\_000: 所有标记的测试集和训练集， 训练集已经随机打乱。 +* train.list and test.list: 训练集和测试集文件列表。 +* dict.txt: 利用训练集生成的字典。 +* labels.txt: neg 0, pos 1, 含义：标签0表示负面的评论，标签1表示正面的评论。 + +### 用户自定义数椐预处理 + +如果你执行其它的用情感分析来分类文本的任务，可以按如下的结构来准备数椐. 我们提供了脚本来构建字典和预处理数椐。所以你只用按下面的结构来组织数椐就行了。 + +``` +dataset +|----train +| |----class1 +| | |----text_files +| |----class2 +| | |----text_files +| | ... +|----test +| |----class1 +| | |----text_files +| |----class2 +| | |----text_files +| | ... +``` +* dataset: 一级目录。 +* train, test: 二级目录。 +* class1,class2,...: 三级目录。 +* text_files: 文本格式的实例文件。 + +所有同目录下的文本实例文件都是同级别的。 每个文本文件包含一个或者多个实例，每一行表示一个实例。 为了充分的随机打乱训练集, 在预处理含有多行数椐的文本文件时参数设置稍有不同, 执行`preprocess.sh`脚本时需要加上`-m True`参数。 tokenizer.perl 默认用来切分单记和标点符号，如果你不需要这个操作，在运行`preprocess.sh`时加上`-t False`参数即可。 + +## 训练模型 + +在这步任务中,我们使用了循环神经网络（RNN）的 LSTM 架构来训练情感分析模型。 引入LSTM模型主要是为了克服消失梯度的问题。 LSTM网络类似于具有隐藏层的标准循环神经网络, 但是隐藏层中的每个普通节点被一个记忆单元替换。 每个记忆单元包含四个主要的元素: 输入门, 具有自循环连接的神经元，忘记门和输出门。 更多的细节可以在文献中找到[4]。 LSTM架构的最大优点是它可以在长时间间隔内记忆信息，而没有短时记忆的损失。在有新的单词来临的每一个时间步骤内，存储在记忆单元区块的历史信息被更新用来迭代的学习单词以合理的序列程现。 + +
![LSTM](../../../doc/demo/sentiment_analysis/lstm.png)
+
图表 1. LSTM [3]
+ +情感分析是自然语言理解中最典型的问题之一。 它的目的是预测在一个序列中表达的情感态度。 通常, ，仅仅是一些关键词，如形容词和副词，在预测序列或段落的情感中起主要作用。然而有些评论上下文非常长，例如 IMDB的数椐集。 我们只所以使用LSTM来执行这个任务是因为其改进的设计并且具有门机制。 首先，它能够从词级到具有可变上下文长度的上下文级别来总结表示。 第二，它可以在句子级别利用可扩展的上下文, 而大多数方法只是利用n-gram级别的知识。第三，它直接学习段落表示，而不是组合上下文级别信息。 + +在本演示中，我们提供两个网络，即双向LSTM和三层堆叠LSTM。 + +#### 双向LSTM + +图2是双向LSTM网络，后面连全连接层和softmax层。 + +
![BiLSTM](../../../doc/demo/sentiment_analysis/bi_lstm.jpg)
+
图 2. Bidirectional-LSTM
+ +#### Stacked-LSTM +图3是三层LSTM结构。图的底部是word embedding(对文档处理后形成的单词向量)。 接下来，连接三个LSTM隐藏层，并且第二个是反向LSTM。然后提取隐藏LSTM层的所有时间步长的最大词向量作为整个序列的表示。 最后，使用具有softmax激活的全连接前馈层来执行分类任务。 更多内容可查看参考文献 [5]。 + +
![StackedLSTM](../../../doc/demo/sentiment_analysis/stacked_lstm.jpg)
+
图 3. Stacked-LSTM for sentiment analysis
+ +**配置** + +进入`demo/sentiment` 目录 , `trainer_config.py` 是一个配置文件的例子, 其中包含算法和网络配置。第一行从`sentiment_net.py`中导出预定义的网络。 + +trainer_config.py: + +```python +from sentiment_net import * + +data_dir = "./data/pre-imdb" +# whether this config is used for test +is_test = get_config_arg('is_test', bool, False) +# whether this config is used for prediction +is_predict = get_config_arg('is_predict', bool, False) +dict_dim, class_dim = sentiment_data(data_dir, is_test, is_predict) + +################## Algorithm Config ##################### + +settings( + batch_size=128, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25 +) + +#################### Network Config ###################### +stacked_lstm_net(dict_dim, class_dim=class_dim, + stacked_num=3, is_predict=is_predict) +#bidirectional_lstm_net(dict_dim, class_dim=class_dim, is_predict=is_predict) +``` + +* **数椐定义**: + * get\_config\_arg(): 获取通过 `--config_args=xx` 设置的命令行参数。 + * 定义训练数椐和测试数椐提供者, 这里使用了PaddlePaddle的Python接口来加载数椐。想了解更多细节可以参考PyDataProvider部分的文档 + +* **算法配置**: + * 使用随机梯度下降（sgd）算法。 + * 使用 adam 优化。 + * 设置batch size大小为128。 + * 设置平均sgd窗口。 + * 设置全局学习率。 +* **网络配置**: + * dict_dim: 获取字典维度。 + * class_dim: 设置类别数，IMDB有两个标签，即正面评价标签和负面评价标签。 + * `stacked_lstm_net`: 预定义网络如图3所示，默认情况下使用此网络 + * `bidirectional_lstm_net`: 预定义网络，如图2所示。 + +**训练** + +首先安装PaddlePaddle。 然后使用下面的脚本 `train.sh` 来开启本地的训练。 + +``` +cd demo/sentiment/ +./train.sh +``` + +train.sh: + +``` +config=trainer_config.py +output=./model_output +paddle train --config=$config \ + --save_dir=$output \ + --job=train \ + --use_gpu=false \ + --trainer_count=4 \ + --num_passes=10 \ + --log_period=20 \ + --dot_period=20 \ + --show_parameter_stats_period=100 \ + --test_all_data_in_one_period=1 \ + 2>&1 | tee 'train.log' +``` + +* \--config=$config: 设置网络配置。 +* \--save\_dir=$output: 设置输出路径以保存训练完成的模型。 +* \--job=train: 设置工作模式为训练。 +* \--use\_gpu=false: 使用CPU训练，如果你安装GPU版本的PaddlePaddle，并想使用GPU来训练设置为true。 +* \--trainer\_count=4:设置线程数（或GPU个数）。 +* \--num\_passes=15: 设置pass，PaddlePaddle中的一个pass意味着对数据集中的所有样本进行一次训练。 +* \--log\_period=20: 每20个batch打印一次日志。 +* \--show\_parameter\_stats\_period=100: 每100个batch打印一次统计信息。 +* \--test\_all_data\_in\_one\_period=1: 每次测试都测试所有数据。 + +如果运行成功，输出日志保存在路径 `demo/sentiment/train.log`中，模型保存在目录`demo/sentiment/model_output/`中。 输出日志说明如下： + +``` +Batch=20 samples=2560 AvgCost=0.681644 CurrentCost=0.681644 Eval: classification_error_evaluator=0.36875 CurrentEval: classification_error_evaluator=0.36875 +... +Pass=0 Batch=196 samples=25000 AvgCost=0.418964 Eval: classification_error_evaluator=0.1922 +Test samples=24999 cost=0.39297 Eval: classification_error_evaluator=0.149406 +``` +- Batch=xx: 表示训练了xx个Batch。 +- samples=xx: 表示训练了xx个样本。。 +- AvgCost=xx: 从第0个batch到当前batch的平均损失。 +- CurrentCost=xx: 最新log_period个batch处理的当前损失。 +- Eval: classification\_error\_evaluator=xx: 表示第0个batch到当前batch的分类错误。 +- CurrentEval: classification\_error\_evaluator: 最新log_period个batch的分类错误。 +- Pass=0: 通过所有训练集一次称为一遍。 0表示第一次经过训练集。 + +默认情况下，我们使用`stacked_lstm_net`网络，当传递相同的样本数时，它的收敛速度比`bidirectional_lstm_net`快。如果要使用双向LSTM，只需删除最后一行中的注释并把“stacked_lstm_net”注释掉。 + +## 测试模型 + +测试模型是指使用训练出的模型评估已标记的验证集。 + +``` +cd demo/sentiment +./test.sh +``` + +test.sh: + +```bash +function get_best_pass() { + cat $1 | grep -Pzo 'Test .*\n.*pass-.*' | \ + sed -r 'N;s/Test.* error=([0-9]+\.[0-9]+).*\n.*pass-([0-9]+)/\1 \2/g' | \ + sort | head -n 1 +} + +log=train.log +LOG=`get_best_pass $log` +LOG=(${LOG}) +evaluate_pass="model_output/pass-${LOG[1]}" + +echo 'evaluating from pass '$evaluate_pass + +model_list=./model.list +touch $model_list | echo $evaluate_pass > $model_list +net_conf=trainer_config.py +paddle train --config=$net_conf \ + --model_list=$model_list \ + --job=test \ + --use_gpu=false \ + --trainer_count=4 \ + --config_args=is_test=1 \ + 2>&1 | tee 'test.log' +``` + +函数`get_best_pass`依据分类错误率获得最佳模型进行测试。 在本示例中，我们默认使用IMDB的测试数据集作为验证。 与训练不同，它需要在这里指定`--job = test`和模型路径，即`--model_list = $model_list`。如果运行成功，日志将保存在“demo / sentiment / test.log”的路径中。例如，在我们的测试中，最好的模型是`model_output / pass-00002`，分类误差是0.115645，如下： + +``` +Pass=0 samples=24999 AvgCost=0.280471 Eval: classification_error_evaluator=0.115645 +``` + +## 预测 + +`predict.py`脚本提供了一个预测接口。在使用它之前请安装PaddlePaddle的python api。 预测IMDB的未标记评论的一个实例如下： + +``` +cd demo/sentiment +./predict.sh +``` +predict.sh: + +``` +#Note the default model is pass-00002, you shold make sure the model path +#exists or change the mode path. +model=model_output/pass-00002/ +config=trainer_config.py +label=data/pre-imdb/labels.list +python predict.py \ + -n $config\ + -w $model \ + -b $label \ + -d data/pre-imdb/dict.txt \ + -i data/aclImdb/test/pos/10007_10.txt +``` + +* `predict.py`: 预测接口脚本。 +* -n $config : 设置网络配置。 +* -w $model: 设置模型路径。 +* -b $label: 设置标签类别字典，这个字典是整数标签和字符串标签的一个对应。 +* -d data/pre-imdb/dict.txt: 设置字典文件。 +* -i data/aclImdb/test/pos/10014_7.txt: 设置一个要预测的示例文件。 + +注意应该确保默认模型路径`model_output / pass-00002`存在或更改为其它模型路径。 + +本示例的预测结果： + +``` +Loading parameters from model_output/pass-00002/ +./data/aclImdb/test/pos/10014_7.txt: predicting label is pos +``` +我们真诚地感谢您的关注，并欢迎您来参与贡献。 + +## 参考文档 +[1] Brendan O'Connor, Ramnath Balasubramanyan, Bryan R. Routledge, and Noah A. Smith. 2010. [From Tweets to Polls: Linking Text Sentiment to Public Opinion Time Series](http://homes.cs.washington.edu/~nasmith/papers/oconnor+balasubramanyan+routledge+smith.icwsm10.pdf). In ICWSM-2010.
+[2] Johan Bollen, Huina Mao, Xiaojun Zeng. 2011. [Twitter mood predicts the stock market](http://arxiv.org/abs/1010.3003), Journal of Computational Science.
+[3] Alex Graves, Marcus Liwicki, Santiago Fernan- dez, Roman Bertolami, Horst Bunke, and Ju ̈rgen Schmidhuber. 2009. [A novel connectionist system for unconstrained handwriting recognition. IEEE Transactions on Pattern Analysis and Machine In- telligence](http://www.cs.toronto.edu/~graves/tpami_2009.pdf), 31(5):855–868.
+[4] Zachary C. Lipton, [A Critical Review of Recurrent Neural Networks for Sequence Learning](http://arxiv.org/abs/1506.00019v1), arXiv:1506.00019.
+[5] Jie Zhou and Wei Xu; [End-to-end Learning of Semantic Role Labeling Using Recurrent Neural Networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf); ACL-IJCNLP 2015.
diff --git a/doc_cn/faq/reduce_min_pool_size.py b/doc_cn/faq/reduce_min_pool_size.py index 2811b134b66b1ec55903d89e3f38a0cef8c9ef8d..5715397cc11e18246b8522fcc5b4f05780c9a0a7 100644 --- a/doc_cn/faq/reduce_min_pool_size.py +++ b/doc_cn/faq/reduce_min_pool_size.py @@ -3,4 +3,4 @@ def process(settings, filename): os.system('shuf %s > %s.shuf' % (filename, filename)) # shuffle before. with open('%s.shuf' % filename, 'r') as f: for line in f: - yield get_sample_from_line(line) \ No newline at end of file + yield get_sample_from_line(line) diff --git a/doc_cn/faq/word2vec_config.py b/doc_cn/faq/word2vec_config.py index e347252476eab670abfa2cf2dc126d96b6e04857..866b40c3d4c96c1213b3f716f29b14dd38763edb 100644 --- a/doc_cn/faq/word2vec_config.py +++ b/doc_cn/faq/word2vec_config.py @@ -1,8 +1,12 @@ -... # the settings and define data provider is omitted. -DICT_DIM=3000 # dictionary dimension. -word_ids=data_layer('word_ids', size=DICT_DIM) +... # the settings and define data provider is omitted. +DICT_DIM = 3000 # dictionary dimension. +word_ids = data_layer('word_ids', size=DICT_DIM) -emb = embedding_layer(input=word_ids, size=256, param_attr=ParamAttr(sparse_update=True)) +emb = embedding_layer( + input=word_ids, size=256, param_attr=ParamAttr(sparse_update=True)) emb_sum = pooling_layer(input=emb, pooling_type=SumPooling()) predict = fc_layer(input=emb_sum, size=DICT_DIM, act=Softmax()) -outputs(classification_cost(input=predict, label=data_layer('label', size=DICT_DIM))) \ No newline at end of file +outputs( + classification_cost( + input=predict, label=data_layer( + 'label', size=DICT_DIM))) diff --git a/doc_cn/faq/word2vec_dataprovider.py b/doc_cn/faq/word2vec_dataprovider.py index a0a39080cece90c6c4096bba4396bfa91b3ef759..ec2753a7d01d7dd4d804c3bed0bac1be9c3fb3d3 100644 --- a/doc_cn/faq/word2vec_dataprovider.py +++ b/doc_cn/faq/word2vec_dataprovider.py @@ -1,8 +1,10 @@ -DICT_DIM=3000 +DICT_DIM = 3000 + + @provider(input_types=[integer_sequence(DICT_DIM), integer_value(DICT_DIM)]) def process(settings, filename): - with open(filename) as f: - # yield word ids to predict inner word id - # such as [28, 29, 10, 4], 4 - # It means the sentance is 28, 29, 4, 10, 4. - yield read_next_from_file(f) \ No newline at end of file + with open(filename) as f: + # yield word ids to predict inner word id + # such as [28, 29, 10, 4], 4 + # It means the sentance is 28, 29, 4, 10, 4. + yield read_next_from_file(f) diff --git a/doc_cn/ui/data_provider/mnist_config.py b/doc_cn/ui/data_provider/mnist_config.py index 7ba344338c374a7f9e7e4faa804e2e124577c0be..39becff03b08f5e75b8503aaf01e782d2b0fb3be 100644 --- a/doc_cn/ui/data_provider/mnist_config.py +++ b/doc_cn/ui/data_provider/mnist_config.py @@ -1,8 +1,9 @@ from paddle.trainer_config_helpers import * -define_py_data_sources2(train_list='train.list', - test_list=None, - module='mnist_provider', - obj='process') +define_py_data_sources2( + train_list='train.list', + test_list=None, + module='mnist_provider', + obj='process') img = data_layer(name='pixel', size=784) label = data_layer(name='label', size=10) diff --git a/doc_cn/ui/data_provider/mnist_provider.dict.py b/doc_cn/ui/data_provider/mnist_provider.dict.py index bf13b56372b56a1e810fad159cd51371ef46c468..2ba0b126a0d6239f84950e130410aaaa6e1f24cd 100644 --- a/doc_cn/ui/data_provider/mnist_provider.dict.py +++ b/doc_cn/ui/data_provider/mnist_provider.dict.py @@ -2,10 +2,9 @@ from paddle.trainer.PyDataProvider2 import * # Define a py data provider -@provider(input_types={ - 'pixel': dense_vector(28 * 28), - 'label': integer_value(10) -}) +@provider( + input_types={'pixel': dense_vector(28 * 28), + 'label': integer_value(10)}) def process(settings, filename): # settings is not used currently. f = open(filename, 'r') # open one of training file diff --git a/doc_cn/ui/data_provider/mnist_provider.py b/doc_cn/ui/data_provider/mnist_provider.py index 92f1915c1072562a174a62b436de8f5b39dab2d4..8b828641d55735e67ca634107d5b239150649651 100644 --- a/doc_cn/ui/data_provider/mnist_provider.py +++ b/doc_cn/ui/data_provider/mnist_provider.py @@ -2,10 +2,7 @@ from paddle.trainer.PyDataProvider2 import * # Define a py data provider -@provider(input_types=[ - dense_vector(28 * 28), - integer_value(10) -]) +@provider(input_types=[dense_vector(28 * 28), integer_value(10)]) def process(settings, filename): # settings is not used currently. f = open(filename, 'r') # open one of training file diff --git a/doc_cn/ui/data_provider/sentimental_config.py b/doc_cn/ui/data_provider/sentimental_config.py index 051f75e32b5c0b1f36d27a54c42db94a4682ce7b..7ce71608a2372b2484ae40ccf01f0621728ddef2 100644 --- a/doc_cn/ui/data_provider/sentimental_config.py +++ b/doc_cn/ui/data_provider/sentimental_config.py @@ -3,9 +3,12 @@ from paddle.trainer_config_helpers import * dictionary = dict() ... # read dictionary from outside -define_py_data_sources2(train_list='train.list', test_list=None, - module='sentimental_provider', obj='process', - # above codes same as mnist sample. - args={ # pass to provider. - 'dictionary': dictionary - }) +define_py_data_sources2( + train_list='train.list', + test_list=None, + module='sentimental_provider', + obj='process', + # above codes same as mnist sample. + args={ # pass to provider. + 'dictionary': dictionary + }) diff --git a/doc_cn/ui/data_provider/sentimental_provider.py b/doc_cn/ui/data_provider/sentimental_provider.py index bda37d7722a0bb98c2c681c790bb308c0e146515..0fb0bb88e95a230f01f18b78ebb37b659c3768f1 100644 --- a/doc_cn/ui/data_provider/sentimental_provider.py +++ b/doc_cn/ui/data_provider/sentimental_provider.py @@ -12,7 +12,8 @@ def on_init(settings, dictionary, **kwargs): # The text is a sequence of integer values, and each value is a word id. # The whole sequence is the sentences that we want to predict its # sentimental. - integer_value(len(dictionary), seq_type=SequenceType), # text input + integer_value( + len(dictionary), seq_type=SequenceType), # text input # label positive/negative integer_value(2) diff --git a/paddle/.common_test_util.sh b/paddle/.common_test_util.sh index dec22e45619fb5d393be96e929a7e301bf266224..dc1525061590808e3cc9c7b606aca5d5d9195a3a 100644 --- a/paddle/.common_test_util.sh +++ b/paddle/.common_test_util.sh @@ -117,4 +117,4 @@ set_port() fi done -} \ No newline at end of file +} diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index cae0f64400a7e618bffb4f7fc6a044011baf04d4..fb3af8ea92feed96a9669bfb29ef7353a256c308 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -17,5 +17,3 @@ endif() if(WITH_SWIG_PY) add_subdirectory(api) endif() - - diff --git a/paddle/api/PaddleAPIPrivate.h b/paddle/api/PaddleAPIPrivate.h index 93cdca8c4beaaad70a40e5899ccd908594425f4f..5ffeff6a9726c7445db36c7c1bec7c74825884a0 100644 --- a/paddle/api/PaddleAPIPrivate.h +++ b/paddle/api/PaddleAPIPrivate.h @@ -65,4 +65,3 @@ struct ArgumentsPrivate { return *(std::shared_ptr*)(rawPtr); } }; - diff --git a/paddle/api/__init__.py b/paddle/api/__init__.py index 7f9e87eee6037666b86420fba194624859d356b3..c90af2ee000d46a032984ee23559e7e99b49ddad 100644 --- a/paddle/api/__init__.py +++ b/paddle/api/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index 05d741f8859ba46893bff49681536d9187a3ed6e..ebe00798e8b7169ecbbef53e287ab4b78334bcf9 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -29,7 +29,10 @@ try: whole_start = "" whole_end = "" - LIB_DIRS = ["math", 'utils', 'parameter', "gserver", "api", "cuda", "pserver", "trainer"] + LIB_DIRS = [ + "math", 'utils', 'parameter', "gserver", "api", "cuda", "pserver", + "trainer" + ] PARENT_LIB_DIRS = ['proto'] class PaddleLDFlag(object): @@ -55,19 +58,20 @@ try: self.curt = CUDA_LIBRARIES def ldflag_str(self): - return " ".join([self.libs_dir_str(), - self.parent_dir_str(), - self.libs_str()]) + return " ".join( + [self.libs_dir_str(), self.parent_dir_str(), self.libs_str()]) def libs_dir_str(self): libdirs = LIB_DIRS - return " ".join(map(lambda x: "-L" + os.path.join(self.paddle_build_dir, x), - libdirs)) + return " ".join( + map(lambda x: "-L" + os.path.join(self.paddle_build_dir, x), + libdirs)) def parent_dir_str(self): libdirs = PARENT_LIB_DIRS - return " ".join(map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x), - libdirs)) + return " ".join( + map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x), + libdirs)) def libs_str(self): libs = [ @@ -113,10 +117,10 @@ try: return cmake_flag elif cmake_flag.startswith("-l"): # normal link command return cmake_flag - elif cmake_flag in ["gflags-shared", - "gflags-static", - "gflags_nothreads-shared", - "gflags_nothreads-static"]: # special for gflags + elif cmake_flag in [ + "gflags-shared", "gflags-static", "gflags_nothreads-shared", + "gflags_nothreads-static" + ]: # special for gflags assert PaddleLDFlag.cmake_bool(self.gflags_location) return self.gflags_location elif len(cmake_flag) != 0: @@ -132,18 +136,22 @@ try: :type cmake_str: str :rtype: bool """ - if cmake_str in ["FALSE", "OFF", "NO"] or cmake_str.endswith("-NOTFOUND"): + if cmake_str in ["FALSE", "OFF", "NO"] or cmake_str.endswith( + "-NOTFOUND"): return False else: return True + def c_flag(self): if self.with_coverage: return ["-fprofile-arcs", "-ftest-coverage", "-O0", "-g"] else: return None except ImportError: + class PaddleLDFlag(object): def ldflag_str(self): pass + def c_flag(self): pass diff --git a/paddle/api/test/CMakeLists.txt b/paddle/api/test/CMakeLists.txt index c4c26e6c03fdff51696f75f4d6a522cff60e7cca..08a0fe96a004d38b81d0bac881da1faeb52685f4 100644 --- a/paddle/api/test/CMakeLists.txt +++ b/paddle/api/test/CMakeLists.txt @@ -1,2 +1,2 @@ add_test(NAME test_swig_api - COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh) \ No newline at end of file + COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh) diff --git a/paddle/api/test/testArguments.py b/paddle/api/test/testArguments.py index daedd2409effccba27ff6818fc2603d3e1665bde..70fb169fd5c43d5768e67ad8e4c62a9f4d302eaf 100644 --- a/paddle/api/test/testArguments.py +++ b/paddle/api/test/testArguments.py @@ -32,7 +32,7 @@ class TestArguments(unittest.TestCase): iv = args.getSlotIds(0) assert isinstance(iv, swig_paddle.IVector) np_arr = iv.toNumpyArrayInplace() - self.assertEqual(np_arr.shape, (6,)) + self.assertEqual(np_arr.shape, (6, )) if __name__ == '__main__': diff --git a/paddle/api/test/testGradientMachine.py b/paddle/api/test/testGradientMachine.py index 59b36a012a239730a1d0a5b239a3ba69f0cee1fb..e12613fbb8a66545dd3ad20d59b0b951e86e8683 100644 --- a/paddle/api/test/testGradientMachine.py +++ b/paddle/api/test/testGradientMachine.py @@ -30,8 +30,8 @@ class TestGradientMachine(unittest.TestCase): self.assertIsNotNone(model_config) machine = swig_paddle.GradientMachine.createByModelConfig( model_config, swig_paddle.CREATE_MODE_NORMAL, - swig_paddle.ParameterOptimizer.create( - opt_config).getParameterTypes()) + swig_paddle.ParameterOptimizer.create(opt_config).getParameterTypes( + )) self.assertIsNotNone(machine) ipt, _ = util.loadMNISTTrainData() output = swig_paddle.Arguments.createArguments(0) @@ -43,7 +43,7 @@ class TestGradientMachine(unittest.TestCase): assert isinstance(param, swig_paddle.Parameter) val = param.getBuf(swig_paddle.PARAMETER_VALUE) assert isinstance(val, swig_paddle.Vector) - arr = numpy.full((len(val),), 0.1, dtype="float32") + arr = numpy.full((len(val), ), 0.1, dtype="float32") val.copyFromNumpyArray(arr) param_config = param.getConfig().toProto() assert isinstance(param_config, diff --git a/paddle/api/test/testMatrix.py b/paddle/api/test/testMatrix.py index 2216ef30a58b0d97bba210bf0edee02a18264076..2160612888b0f7ed6b504e5fc5933dfb3781f167 100644 --- a/paddle/api/test/testMatrix.py +++ b/paddle/api/test/testMatrix.py @@ -69,7 +69,8 @@ class TestMatrix(unittest.TestCase): def test_numpy(self): numpy_mat = np.matrix([[1, 2], [3, 4], [5, 6]], dtype="float32") m = swig_paddle.Matrix.createCpuDenseFromNumpy(numpy_mat) - self.assertEqual((int(m.getHeight()), int(m.getWidth())), numpy_mat.shape) + self.assertEqual((int(m.getHeight()), int(m.getWidth())), + numpy_mat.shape) # the numpy matrix and paddle matrix shared the same memory. numpy_mat[0, 1] = 342.23 diff --git a/paddle/api/test/testTrain.py b/paddle/api/test/testTrain.py index 7759118a3d9d108f0c05d985ac74a5122799ccb4..a3ba4eaaa69b39b75e7ece3095b6f236c1248d41 100644 --- a/paddle/api/test/testTrain.py +++ b/paddle/api/test/testTrain.py @@ -98,7 +98,8 @@ def main(): cost_vec = outArgs.getSlotValue(0) assert isinstance(cost_vec, swig_paddle.Matrix) cost_vec = cost_vec.copyToNumpyMat() - print 'Finish Batch', batch_id, 'with cost ', cost_vec.sum() / batch_size + print 'Finish Batch', batch_id, 'with cost ', cost_vec.sum( + ) / batch_size batch_id += 1 for optimizer in optimizers: diff --git a/paddle/api/test/testTrainConfig.py b/paddle/api/test/testTrainConfig.py index 22148e31915da0c21609fe0694274cfaee4b3950..77e0cd37d566d2571fada76b9948a9b0616ad044 100644 --- a/paddle/api/test/testTrainConfig.py +++ b/paddle/api/test/testTrainConfig.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=100, - learning_method=AdamOptimizer() -) +settings(batch_size=100, learning_method=AdamOptimizer()) din = data_layer(name='input', size=784) diff --git a/paddle/api/test/testTrainer.py b/paddle/api/test/testTrainer.py index da69a60f84f4d7c6fad54fc116a31b54ef162f60..edd5a2da5785c405b46c2559ee93837ac68d7c3a 100644 --- a/paddle/api/test/testTrainer.py +++ b/paddle/api/test/testTrainer.py @@ -17,9 +17,9 @@ from paddle.trainer.config_parser import logger from py_paddle import swig_paddle import util + def main(): - trainer_config = parse_config( - "./testTrainConfig.py", "") + trainer_config = parse_config("./testTrainConfig.py", "") model = swig_paddle.GradientMachine.createFromConfigProto( trainer_config.model_config) trainer = swig_paddle.Trainer.create(trainer_config, model) @@ -56,7 +56,7 @@ def main(): logger.info('test cost=%f' % (cost / num)) trainer.finishTrain() - + if __name__ == '__main__': swig_paddle.initPaddle("--use_gpu=0", "--trainer_count=1") diff --git a/paddle/api/test/testVector.py b/paddle/api/test/testVector.py index f5b5d0e32e4208e7becb9755d1aed131f52ff146..5226df79eea3bedbf2b5b6f5fa684cc99a194f7c 100644 --- a/paddle/api/test/testVector.py +++ b/paddle/api/test/testVector.py @@ -112,5 +112,6 @@ class TestVector(unittest.TestCase): if __name__ == '__main__': - swig_paddle.initPaddle("--use_gpu=1" if swig_paddle.isGpuVersion() else "--use_gpu=0") + swig_paddle.initPaddle("--use_gpu=1" + if swig_paddle.isGpuVersion() else "--use_gpu=0") unittest.main() diff --git a/paddle/cuda/include/hl_base.h b/paddle/cuda/include/hl_base.h index 02fa6bc3ace32ff5ffe51dcce9c49757a990a9b2..9f80898a1f927a0e8bbf86108567a04ccecc38f5 100644 --- a/paddle/cuda/include/hl_base.h +++ b/paddle/cuda/include/hl_base.h @@ -254,4 +254,3 @@ extern __thread cudaStream_t default_stream; #endif /* __NVCC__ */ #endif /* HL_BASE_H_ */ - diff --git a/paddle/cuda/include/hl_cnn.h b/paddle/cuda/include/hl_cnn.h index ac35727ac28c7ee7a41d7d3d93d7c18288950a41..70b5be6fda2509853029a68d31129df28d580942 100644 --- a/paddle/cuda/include/hl_cnn.h +++ b/paddle/cuda/include/hl_cnn.h @@ -91,6 +91,7 @@ extern void hl_expand_feature2col( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] tgtData output data. + * @param[in] tgtStride stride between output data samples. * */ extern void hl_maxpool_forward( @@ -100,7 +101,8 @@ extern void hl_maxpool_forward( const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData); + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride); /** * @brief Maximum pool backward. @@ -123,6 +125,7 @@ extern void hl_maxpool_forward( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] targetGrad output grad. + * @param[in] outStride stride between output data samples. * */ extern void hl_maxpool_backward( @@ -135,7 +138,7 @@ extern void hl_maxpool_backward( const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, - real* targetGrad); + real* targetGrad, const int outStride); /** * @brief Averge pool forward. @@ -154,6 +157,7 @@ extern void hl_maxpool_backward( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] tgtData output data. + * @param[in] tgtStride stride between output data samples. * */ extern void hl_avgpool_forward( @@ -163,7 +167,8 @@ extern void hl_avgpool_forward( const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData); + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride); /** * @brief Maximum pool backward. @@ -184,6 +189,7 @@ extern void hl_avgpool_forward( * @param[in] scaleA scale. * @param[in] scaleB scale. * @param[out] backGrad output grad. + * @param[in] outStride stride between output data samples. * */ extern void hl_avgpool_backward( @@ -195,7 +201,7 @@ extern void hl_avgpool_backward( const int strideH, const int strideW, int paddingH, int paddingW, real scaleA, real scaleB, - real* backGrad); + real* backGrad, const int outStride); /** * @brief Cross-map-respose normalize forward. diff --git a/paddle/cuda/include/hl_matrix.h b/paddle/cuda/include/hl_matrix.h index 71e8f8e3a60c9ff340f36c5057a22cecc112fd48..6195e30b9974d3ad092b4cf604e6b74fa481835c 100644 --- a/paddle/cuda/include/hl_matrix.h +++ b/paddle/cuda/include/hl_matrix.h @@ -126,6 +126,36 @@ extern void hl_matrix_cross_entropy_bp(real* grad_d, int dimM, int dimN); +/** + * @brief Matrix multi-binary label cross entropy + * + * @param[in] output input matrix (M x N). + * @param[out] entropy output matrix (M x 1). + * @param[in] mat input sparse matrix. + * @param[in] dimM matrix height. + * @param[in] dimN matrix width. + */ +extern void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s mat, + int dimM, + int dimN); + +/** + * @brief Matrix multi-binary label cross entropy backprop + * + * @param[in] output input matrix (M x N). + * @param[out] grad output matrix (M x N). + * @param[in] mat input sparse matrix. + * @param[in] dimM matrix height. + * @param[in] dimN matrix width. + */ +extern void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s mat, + int dimM, + int dimN); + /** * @brief Matrix zero memory. * diff --git a/paddle/cuda/include/stub/hl_cnn_stub.h b/paddle/cuda/include/stub/hl_cnn_stub.h index 50fddce584252b459b1146b8528e2918416aff95..c6f32ad337705ff938b7b370a4785dc7f4393041 100644 --- a/paddle/cuda/include/stub/hl_cnn_stub.h +++ b/paddle/cuda/include/stub/hl_cnn_stub.h @@ -44,7 +44,8 @@ inline void hl_maxpool_forward( const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData) {} + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) {} inline void hl_maxpool_backward( const int frameCnt, const real* inputData, @@ -56,7 +57,7 @@ inline void hl_maxpool_backward( const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, - real* targetGrad) {} + real* targetGrad, const int outStride) {} inline void hl_avgpool_forward( const int frameCnt, const real* inputData, @@ -65,7 +66,8 @@ inline void hl_avgpool_forward( const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData) {} + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) {} inline void hl_avgpool_backward( const int frameCnt, const real* outGrad, @@ -76,7 +78,7 @@ inline void hl_avgpool_backward( const int strideH, const int strideW, int paddingH, int paddingW, real scaleA, real scaleB, - real* backGrad) {} + real* backGrad, const int outStride) {} inline void hl_CMRNorm_forward( size_t frameCnt, const real* in, real* scale, real* out, diff --git a/paddle/cuda/include/stub/hl_cuda_cudnn_stub.h b/paddle/cuda/include/stub/hl_cuda_cudnn_stub.h index 34c173908246e4a48c327c8aa58730756bbc72b7..b96804afd86ba5e8c7b7eed7eb768295b4e23096 100644 --- a/paddle/cuda/include/stub/hl_cuda_cudnn_stub.h +++ b/paddle/cuda/include/stub/hl_cuda_cudnn_stub.h @@ -199,4 +199,3 @@ inline void hl_batch_norm_backward(hl_tensor_descriptor inputDesc, real *savedInvVar) {} #endif // HL_CUDA_CUDNN_STUB_H_ - diff --git a/paddle/cuda/include/stub/hl_matrix_stub.h b/paddle/cuda/include/stub/hl_matrix_stub.h index e37b1275432caae29b14e95658e3db291632a672..76cac2e57769301fee2e5979e2685976daf35441 100644 --- a/paddle/cuda/include/stub/hl_matrix_stub.h +++ b/paddle/cuda/include/stub/hl_matrix_stub.h @@ -57,6 +57,18 @@ inline void hl_matrix_cross_entropy_bp(real* grad_d, int dimM, int dimN) {} +inline void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s mat, + int dimM, + int dimN) {} + +inline void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s mat, + int dimM, + int dimN) {} + inline void hl_matrix_zero_mem(real* data, int num) {} inline void hl_param_relu_forward(real* output, diff --git a/paddle/cuda/src/avx_mathfun.h b/paddle/cuda/src/avx_mathfun.h index 808c2508d1a1a09fb25f052047d6b0539cad8df2..2922d4dc2937662d66fb2433f4883448ba21fa3f 100644 --- a/paddle/cuda/src/avx_mathfun.h +++ b/paddle/cuda/src/avx_mathfun.h @@ -718,4 +718,3 @@ void sincos256_ps(v8sf x, v8sf *s, v8sf *c) { *s = _mm256_xor_ps(xmm1, sign_bit_sin); *c = _mm256_xor_ps(xmm2, sign_bit_cos); } - diff --git a/paddle/cuda/src/hl_cuda_cnn.cu b/paddle/cuda/src/hl_cuda_cnn.cu index 9eec44f77f27a0bf29d3b68260d663a5687d1b0c..ae387a8bc0e0791995810df9e5f2556264d869b1 100644 --- a/paddle/cuda/src/hl_cuda_cnn.cu +++ b/paddle/cuda/src/hl_cuda_cnn.cu @@ -152,7 +152,7 @@ __global__ void KeMaxPoolForward(const int nthreads, const real* inputData, const int ksizeW, const int ksizeH, const int strideH, const int strideW, const int offsetH, const int offsetW, - real* tgtData) { + real* tgtData, const int tgtStride) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % pooledW; @@ -173,7 +173,9 @@ __global__ void KeMaxPoolForward(const int nthreads, const real* inputData, maxval = inputData[h * width + w]; } } - tgtData[index] = maxval; + int tgtIndex = index % (pooledW * pooledH * channels) + + frameNum * tgtStride; + tgtData[tgtIndex] = maxval; } } @@ -184,7 +186,7 @@ void hl_maxpool_forward(const int frameCnt, const real* inputData, const int sizeX, const int sizeY, const int strideH, const int strideW, const int paddingH, const int paddingW, - real* tgtData) { + real* tgtData, const int tgtStride) { int num_kernels = pooledH * pooledW * channels * frameCnt; int blocks = (num_kernels + 1024 - 1) / 1024; @@ -194,7 +196,7 @@ void hl_maxpool_forward(const int frameCnt, const real* inputData, KeMaxPoolForward<<< grid, threads, 0, STREAM_DEFAULT >>> (num_kernels, inputData, channels, height, width, pooledH, pooledW, sizeX, sizeY, strideH, strideW, - paddingH, paddingW, tgtData); + paddingH, paddingW, tgtData, tgtStride); CHECK_SYNC("hl_maxpool_forward failed"); } @@ -207,7 +209,7 @@ __global__ void KeMaxPoolBackward(const int nthreads, const real* inputData, const int strideH, const int strideW, const int padH, const int padW, real scaleA, real scaleB, - real* targetGrad) { + real* targetGrad, const int outStride) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { // find out the local index @@ -223,8 +225,8 @@ __global__ void KeMaxPoolBackward(const int nthreads, const real* inputData, int pwend = offsetW >= 0 ? min(offsetW / strideW + 1, pooledW) : 0; real gradient = 0; real input = inputData[index]; - outData += (frameNum * channels + offsetC) * pooledH * pooledW; - outGrad += (frameNum * channels + offsetC) * pooledH * pooledW; + outData += (frameNum * outStride + offsetC * pooledH * pooledW); + outGrad += (frameNum * outStride + offsetC * pooledH * pooledW); for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { if (input == outData[ph * pooledW + pw]) { @@ -246,7 +248,7 @@ void hl_maxpool_backward(const int frameCnt, const real* inputData, const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, - real* targetGrad) { + real* targetGrad, const int outStride) { int num_kernels = height * width * channels * frameCnt; int blocks = (num_kernels + 1024 - 1) / 1024; @@ -257,7 +259,7 @@ void hl_maxpool_backward(const int frameCnt, const real* inputData, strideH, strideW, paddingH, paddingW, scaleA, scaleB, - targetGrad); + targetGrad, outStride); CHECK_SYNC("hl_maxpool_backward"); } @@ -268,7 +270,7 @@ __global__ void KeAvgPoolForward(const int nthreads, const real* inputData, const int sizeX, const int sizeY, const int strideH, const int strideW, const int padH, const int padW, - real* tgtData) { + real* tgtData, const int tgtStride) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % pooledW; @@ -293,7 +295,9 @@ __global__ void KeAvgPoolForward(const int nthreads, const real* inputData, aveval += inputData[h * width + w]; } } - tgtData[index] = aveval / pool_size; + int tgtIndex = index % (pooledW * pooledH * channels) + + frameNum * tgtStride; + tgtData[tgtIndex] = aveval / pool_size; } } @@ -303,14 +307,15 @@ void hl_avgpool_forward(const int frameCnt, const real* inputData, const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData) { + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) { int num_kernels = pooledH * pooledW * channels * frameCnt; int blocks = (num_kernels + 1024 - 1) / 1024; KeAvgPoolForward<<< blocks, 1024, 0, STREAM_DEFAULT >>> (num_kernels, inputData, channels, height, width, pooledH, pooledW, sizeX, sizeY, strideH, strideW, - paddingH, paddingW, tgtData); + paddingH, paddingW, tgtData, tgtStride); CHECK_SYNC("hl_avgpool_forward failed"); } @@ -322,7 +327,7 @@ __global__ void KeAvgPoolBackward(const int nthreads, const real* outGrad, const int strideH, const int strideW, const int padH, const int padW, real scaleA, real scaleB, - real* tgtGrad) { + real* tgtGrad, const int outStride) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int offsetW = index % width + padW; @@ -335,7 +340,8 @@ __global__ void KeAvgPoolBackward(const int nthreads, const real* outGrad, int phend = offsetH >= 0 ? min(offsetH / strideH + 1, pooledH) : 0; int pwend = offsetW >= 0 ? min(offsetW / strideW + 1, pooledW) : 0; real gradient = 0; - outGrad += (frameNum * channels + offsetC) * pooledH * pooledW; + outGrad += (frameNum * outStride + offsetC * pooledH * pooledW); + for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { @@ -360,7 +366,7 @@ void hl_avgpool_backward(const int frameCnt, const real* outGrad, const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, - real* backGrad) { + real* backGrad, const int outStride) { int num_kernels = height * width * channels * frameCnt; int blocks = (num_kernels + 1024 - 1) / 1024; @@ -370,7 +376,7 @@ void hl_avgpool_backward(const int frameCnt, const real* outGrad, strideH, strideW, paddingH, paddingW, scaleA, scaleB, - backGrad); + backGrad, outStride); CHECK_SYNC("hl_avgpool_backward failed"); } diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index 3df9f63f9e4b79d61a818b2af49a4d9dfd84a9ab..0b7cd3375671d58464dac93458ec6659add8b730 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -18,6 +18,7 @@ limitations under the License. */ #include "hl_matrix_ops.cuh" #include "hl_matrix_apply.cuh" #include "hl_sequence.h" +#include "hl_sparse.ph" #include "paddle/utils/Logging.h" #include "hl_device_functions.cuh" #include "hl_gpu_matrix_kernel.cuh" @@ -317,6 +318,85 @@ void hl_matrix_classification_error(real* A_d, CHECK_SYNC("hl_matrix_classification_error"); } +__global__ void KeMatrixMultiBinaryCrossEntropy(real* output, + real* entropy, + int* row, + int* col, + int dimM, + int dimN) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < dimM) { + for (int i = 0; i < dimN; i ++) { + entropy[index] -= log(1 - output[index * dimN + i]); + } + int *row_col = col + row[index]; + int col_num = row[index + 1] - row[index]; + for (int i = 0; i < col_num; i ++) { + real o = output[index * dimN + row_col[i]]; + entropy[index] -= log(o / (1 - o)); + } + } +} + +void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s csr_mat, + int dimM, + int dimN) { + CHECK_NOTNULL(output); + CHECK_NOTNULL(entropy); + CHECK_NOTNULL(csr_mat); + CHECK_EQ(csr_mat->format, HL_SPARSE_CSR); + int n_threads = 1024; + int blocks = (dimM + n_threads - 1) / n_threads; + dim3 threads(n_threads); + dim3 grid(blocks); + hl_csr_matrix mat = (hl_csr_matrix)(csr_mat->matrix); + KeMatrixMultiBinaryCrossEntropy<<< grid, threads, 0, STREAM_DEFAULT >>> + (output, entropy, mat->csr_row, mat->csr_col, dimM, dimN); + CHECK_SYNC("hl_matrix_multi_binary_cross_entropy failed"); +} + +__global__ void KeMatrixMultiBinaryCrossEntropyBp(real* output, + real* grad, + int* row, + int* col, + int dimM, + int dimN) { + int row_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (row_idx < dimM) { + for (int i = 0; i < dimN; i ++) { + int index = row_idx * dimN + i; + grad[index] += 1.0 / (1 - output[index]); + } + int col_num = row[row_idx + 1] - row[row_idx]; + int *row_col = col + row[row_idx]; + for (int i = 0; i < col_num; i ++) { + int index = row_idx * dimN + row_col[i]; + grad[index] -= 1.0 / (output[index] * (1 - output[index])); + } + } +} + +void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s csr_mat, + int dimM, + int dimN) { + CHECK_NOTNULL(output); + CHECK_NOTNULL(grad); + CHECK_NOTNULL(csr_mat); + CHECK_EQ(csr_mat->format, HL_SPARSE_CSR); + int n_threads = 1024; + int blocks = (dimM + n_threads - 1) / n_threads; + dim3 threads(n_threads); + dim3 grid(blocks); + hl_csr_matrix mat = (hl_csr_matrix)(csr_mat->matrix); + KeMatrixMultiBinaryCrossEntropyBp<<< grid, threads, 0, STREAM_DEFAULT >>> + (output, grad, mat->csr_row, mat->csr_col, dimM, dimN); + CHECK_SYNC("hl_matrix_multi_binary_cross_entropy_bp failed"); +} + __global__ void KeMatrixCrossEntropy(real* O, real* E, int* label, @@ -685,7 +765,7 @@ __global__ void KeMatrixAddSharedBias(real* A, int dim = N / channel; if (index < M * N) { int i = index % N; - i = i / dim; + i = i / dim; A[index] += scale * B[i]; } } @@ -713,7 +793,7 @@ __global__ void KeMatrixCollectSharedBias(real *B, const int dim, const int limit, real scale) { - if (dim < limit) { + if (dim < limit) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < channel) { real sum = 0.0; diff --git a/paddle/gserver/layers/ConcatenateLayer.cpp b/paddle/gserver/layers/ConcatenateLayer.cpp index bb6709b8df330b5f06a9df9887f54644c82d2878..a986ec10b4a01c8cc87b067d13e76b8c456bda34 100644 --- a/paddle/gserver/layers/ConcatenateLayer.cpp +++ b/paddle/gserver/layers/ConcatenateLayer.cpp @@ -160,7 +160,9 @@ void ConcatenateLayer2::forward(PassType passType) { size_t startCol = projCol_[i].first; size_t endCol = projCol_[i].second; projOutput_[i].value = output_.value->subColMatrix(startCol, endCol); - projOutput_[i].grad = output_.grad->subColMatrix(startCol, endCol); + if (output_.grad) { + projOutput_[i].grad = output_.grad->subColMatrix(startCol, endCol); + } } { diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 42ff0b70d86f788d58d56854a778d61e2af53e06..6bc3b3b801796a227a7b767c8da048a3ccf88827 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -14,12 +14,15 @@ limitations under the License. */ #include "paddle/utils/Logging.h" #include "ConvBaseLayer.h" +#include "paddle/math/MathUtils.h" namespace paddle { bool ConvBaseLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { /* Initialize the basic parent class */ Layer::init(layerMap, parameterMap); + isDeconv_ = (config_.type() == "exconv" || config_.type() == "cudnn_conv") + ? false : true; /* Initialize the convolutional layer parameter */ numFilters_ = config_.num_filters(); @@ -42,8 +45,20 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, outputW_.push_back(conf.output_x()); } + CHECK(inputLayers_.size() == parameters_.size()); + for (size_t i = 0; i < inputLayers_.size(); i++) { + size_t height, width; + height = filterPixels_[i] * filterChannels_[i]; + width = (!isDeconv_) ? numFilters_ : channels_[i]; + + // create a new weight + CHECK_EQ(parameters_[i]->getSize(), width * height); + Weight* w = new Weight(height, width, parameters_[i]); + weights_.emplace_back(w); + } + /* initialize the biases_ */ - if (biasParameter_.get() != NULL) { + if (biasParameter_.get()) { if (sharedBiases_) { CHECK_EQ((size_t)numFilters_, biasParameter_->getSize()); biases_ = @@ -70,23 +85,48 @@ size_t ConvBaseLayer::calOutputSize() { clearAndReserve(&outputH_); clearAndReserve(&outputW_); size_t layerSize = 0; - for (size_t i = 0; i < inputLayers_.size(); i++) { - imgSizeH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); - imgSizeW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); - if (imgSizeH_[i] == 0) - imgSizeH_[i] = config_.inputs(i).conv_conf().img_size(); - if (imgSizeW_[i] == 0) - imgSizeW_[i] = config_.inputs(i).conv_conf().img_size(); - outputH_.push_back(outputSize(imgSizeH_[i], filterSizeY_[i], paddingY_[i], - strideY_[i], caffeMode_)); - outputW_.push_back(outputSize(imgSizeW_[i], filterSize_[i], padding_[i], - stride_[i], caffeMode_)); - CHECK_EQ(outputH_[i], outputH_[0]); - CHECK_EQ(outputW_[i], outputW_[0]); + + auto setLayerSize = [&](IntV& inH, IntV& inW, IntV& outH, IntV& outW) { + for (size_t i = 0; i < inputLayers_.size(); i++) { + inH.push_back(inputLayers_[i]->getOutput().getFrameHeight()); + inW.push_back(inputLayers_[i]->getOutput().getFrameWidth()); + if (isDeconv_) { + if (inH[i] == 0) + inH[i] = config_.inputs(i).conv_conf().output_x(); + if (inW[i] == 0) + inW[i] = config_.inputs(i).conv_conf().output_x(); + outH.push_back( + imageSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i], + caffeMode_)); + outW.push_back( + imageSize(inW[i], filterSize_[i], padding_[i], stride_[i], + caffeMode_)); + } else { + if (inH[i] == 0) + inH[i] = config_.inputs(i).conv_conf().img_size(); + if (inW[i] == 0) + inW[i] = config_.inputs(i).conv_conf().img_size(); + outH.push_back( + outputSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i], + caffeMode_)); + outW.push_back( + outputSize(inW[i], filterSize_[i], padding_[i], stride_[i], + caffeMode_)); + } + CHECK_EQ(outH[i], outH[0]); + CHECK_EQ(outW[i], outW[0]); + } + getOutput().setFrameHeight(outH[0]); + getOutput().setFrameWidth(outW[0]); + layerSize = outH[0] * outW[0] * size_t(numFilters_); + }; + + if (isDeconv_) { + setLayerSize(outputH_, outputW_, imgSizeH_, imgSizeW_); + } else { + setLayerSize(imgSizeH_, imgSizeW_, outputH_, outputW_); } - getOutput().setFrameHeight(outputH_[0]); - getOutput().setFrameWidth(outputW_[0]); - layerSize = outputH_[0] * outputW_[0] * size_t(numFilters_); + return layerSize; } diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index e660a6d6f50acf8286dfd6fc795e8a03ce3ba604..b80cab899585e7bd93bfc86d8afa116d343d36d7 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -28,6 +28,9 @@ class ConvBaseLayer : public Layer { protected: typedef std::vector IntV; + /// True if it's deconv layer, false if it's convolution layer + bool isDeconv_; + /// The number of filters. int numFilters_; /// The x dimension of the padding. diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index 14ff8510f7b19dc24b7b1ba603485488ddd4979d..3c2df52fed4f86675ce8f1ead6a3b66e4babde34 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -462,25 +462,43 @@ bool MultiBinaryLabelCrossEntropy::init(const LayerMap& layerMap, void MultiBinaryLabelCrossEntropy::forwardImp(Matrix& output, Argument& label, Matrix& target) { - if (dynamic_cast(label.value.get()) || - dynamic_cast(label.value.get())) { - target.multiBinaryLabelCrossEntropy(output, *label.value); + MatrixPtr value = nullptr; + if (label.ids) { + CHECK(!label.value); + value = label.ids->toOneHotSparseMatrix(output.getWidth(), useGpu_); + } else { + CHECK(label.value); + value = label.value; + } + + if (dynamic_cast(value.get()) || + dynamic_cast(value.get())) { + target.multiBinaryLabelCrossEntropy(output, *value); } else { Matrix::resizeOrCreate(targetPerDim_, output.getHeight(), output.getWidth(), false, useGpu_); - targetPerDim_->binaryLabelCrossEntropy(output, *label.value); + targetPerDim_->binaryLabelCrossEntropy(output, *value); targetPerDim_->rowSum(target); } } void MultiBinaryLabelCrossEntropy::backwardImp( Matrix& output, Argument& label, Matrix& outputG) { - if (dynamic_cast(label.value.get()) || - dynamic_cast(label.value.get())) { - outputG.multiBinaryLabelCrossEntropyBp(output, *label.value); + MatrixPtr value = nullptr; + if (label.ids) { + CHECK(!value); + value = label.ids->toOneHotSparseMatrix(output.getWidth(), useGpu_); } else { - outputG.binaryLabelCrossEntropyBp(output, *label.value); + CHECK(label.value); + value = label.value; + } + + if (dynamic_cast(value.get()) || + dynamic_cast(value.get())) { + outputG.multiBinaryLabelCrossEntropyBp(output, *value); + } else { + outputG.binaryLabelCrossEntropyBp(output, *value); } } @@ -562,4 +580,39 @@ void HuberTwoClass::backwardImpIn( } } +/** + * This cost layer compute the sum of its input as loss. + * \f[ + * o(i) = \sum_{j=1}^D y_{ij} + * \f] + */ +class SumCostLayer : public Layer { +public: + explicit SumCostLayer(const LayerConfig& config) : Layer(config) {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap) { + bool ret = Layer::init(layerMap, parameterMap); + if (!ret) return ret; + CHECK_EQ(inputLayers_.size(), 1UL); + return true; + } + + virtual void forward(PassType passType) { + Layer::forward(passType); + const MatrixPtr& input = getInputValue(0); + + /* malloc memory for the output_ if necessary */ + int batchSize = input->getHeight(); + int size = 1; + resizeOutput(batchSize, size); + output_.value->sumRows(*input, /* scaleSum= */1, /* scaleDest= */0); + } + + virtual void backward(const UpdateCallback& callback = nullptr) { + getInputGrad(0)->add((real)1); + } +}; + +REGISTER_LAYER(sum_cost, SumCostLayer); + } // namespace paddle diff --git a/paddle/gserver/layers/CostLayer.h b/paddle/gserver/layers/CostLayer.h index b464e16737ae561dce6e7d4f16a4dd61f73204e0..f263c688213ae6a83d5db4a1025aa252344dfab8 100644 --- a/paddle/gserver/layers/CostLayer.h +++ b/paddle/gserver/layers/CostLayer.h @@ -129,7 +129,7 @@ protected: * This cost layer compute Euclidean (L2) loss for real-valued regression * tasks. * \f[ - * L = \frac{1}{2N} \sum_{i=1}^N {|| \hat{y}_i - y_i||_2^2} + * L = \sum_{i=1}^N {|| \hat{y}_i - y_i||_2^2} * \f] */ class SumOfSquaresCostLayer : public CostLayer { diff --git a/paddle/gserver/layers/ExpandConvBaseLayer.cpp b/paddle/gserver/layers/ExpandConvBaseLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0bab0ca764f4fea7dc37f0eae096de1a79c9df21 --- /dev/null +++ b/paddle/gserver/layers/ExpandConvBaseLayer.cpp @@ -0,0 +1,263 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "ExpandConvBaseLayer.h" + +#include "paddle/utils/Logging.h" +namespace paddle { + +bool ExpandConvBaseLayer::init(const LayerMap &layerMap, + const ParameterMap ¶meterMap) { + /* Initialize the basic convolutional parent class */ + ConvBaseLayer::init(layerMap, parameterMap); + + /* The class fields channels_ and numFilters_ are the same as in the config + * i.e., channels_ is the for the input and numFilters_ is for the output + * + * But in order for the variables in convTrans having the same semantic + * meaning as in conv, we need to swap channels_ and numFilters here for + * convTrans, and in other functions too. + * */ + int channel; + int numFilters; + /* Initialize the projection */ + for (auto &inputConfig : config_.inputs()) { + const ConvConfig &conf = inputConfig.conv_conf(); + numFilters = isDeconv_ ? conf.channels() : numFilters_; + subM_.push_back(numFilters / conf.groups()); + subN_.push_back(conf.output_x() * conf.output_x()); + channel = isDeconv_ ? numFilters_ : conf.channels(); + subK_.push_back(channel * conf.filter_size() * conf.filter_size() / + conf.groups()); + /* Consistent caffe mode for multiple input */ + caffeMode_ = conf.caffe_mode(); + } + + getOutputSize(); + + return true; +} + +size_t ExpandConvBaseLayer::getOutputSize() { + CHECK_NE(inputLayers_.size(), 0UL); + size_t layerSize = ConvBaseLayer::calOutputSize(); + subN_.clear(); + for (size_t i = 0; i < inputLayers_.size(); i++) { + subN_.push_back(outputH_[i] * outputW_[i]); + } + return layerSize; +} + +void ExpandConvBaseLayer::resetExpandInput(size_t height, size_t width) { + Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); +} + +void ExpandConvBaseLayer::addSharedBias() { + size_t mapW = getOutputSize() / numFilters_; + size_t mapH = getOutputValue()->getElementCnt() / mapW; + MatrixPtr out = + Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); + + Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); + + out->transpose(transOutValue_, false); // false means no memory allocation + transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, + numFilters_); + + MatrixPtr bias = + Matrix::create(biases_->getW()->getData(), 1, + biases_->getW()->getElementCnt(), false, useGpu_); + transOutValue_->addBias(*bias, 1.0f); + + transOutValue_->reshape(mapW, mapH); + transOutValue_->transpose(out, false); // false means no memory allocation + + out->clear(); + bias->clear(); +} + +void ExpandConvBaseLayer::addUnsharedBias() { + MatrixPtr outValue = getOutputValue(); + MatrixPtr bias = + Matrix::create(biases_->getW()->getData(), 1, + biases_->getW()->getElementCnt(), false, useGpu_); + outValue->addBias(*bias, 1.0f); +} + + +void ExpandConvBaseLayer::expandOneFrame(MatrixPtr image, size_t startIdx, + int inIdx) { + int channel = isDeconv_ ? numFilters_ : channels_[inIdx]; + + resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); + real *imgData = image->getData() + startIdx * image->getWidth(); + MatrixPtr imageTmp = Matrix::create( + imgData, 1, imgSizeH_[inIdx] * imgSizeW_[inIdx] * channel, false, + useGpu_); + expandInput_->convExpand(*imageTmp, imgSizeH_[inIdx], imgSizeW_[inIdx], + channel, filterSize_[inIdx], + filterSize_[inIdx], stride_[inIdx], stride_[inIdx], + padding_[inIdx], padding_[inIdx], + outputH_[inIdx], outputW_[inIdx]); + imageTmp->clear(); +} + +void ExpandConvBaseLayer::expandFwdOnce(MatrixPtr image, MatrixPtr out, + int inIdx, int startIdx) { + int subM = subM_[inIdx]; + int subN = subN_[inIdx]; + int subK = subK_[inIdx]; + + expandOneFrame(image, startIdx, inIdx); + + int numFilters = isDeconv_ ? channels_[inIdx] : numFilters_; + + real *outData = + out->getData() + startIdx * subN * numFilters; + + real *wgtData = weights_[inIdx]->getW()->getData(); + real *expInData = expandInput_->getData(); + for (int g = 0; g < groups_[inIdx]; ++g) { + MatrixPtr A = + Matrix::create(wgtData, subK, subM, true, useGpu_); // mark transpose + MatrixPtr B = Matrix::create(expInData, subK, subN, false, useGpu_); + MatrixPtr C = Matrix::create(outData, subM, subN, false, useGpu_); + C->mul(A, B, 1, 1); + + A->clear(); + B->clear(); + C->clear(); + wgtData += subK * subM; + expInData += subK * subN; + outData += subM * subN; + } +} + +void ExpandConvBaseLayer::bpropActs(MatrixPtr out, MatrixPtr image, + int inpIdx) { + int channel = isDeconv_ ? numFilters_ : channels_[inpIdx]; + + int subM = subM_[inpIdx]; + int subN = subN_[inpIdx]; + int subK = subK_[inpIdx]; + size_t batchSize = image->getHeight(); + + /* reset the expand-grad memory */ + resetExpandInput(subK * groups_[inpIdx], subN); + + real *localGradData = out->getData(); + real *tgtGradData = image->getData(); + for (size_t n = 0; n < batchSize; n++) { + real *wgtData = weights_[inpIdx]->getW()->getData(); + real *expandInData = expandInput_->getData(); + + for (int g = 0; g < groups_[inpIdx]; g++) { + // create temporary matrix + MatrixPtr C = Matrix::create(expandInData, subK, subN, false, useGpu_); + MatrixPtr B = Matrix::create(localGradData, subM, subN, false, useGpu_); + MatrixPtr A = Matrix::create(wgtData, subK, subM, false, useGpu_); + C->mul(A, B); // mul + + // clear the temporary matrix + A->clear(); + B->clear(); + C->clear(); + + expandInData += subK * subN; + localGradData += subM * subN; + wgtData += subK * subM; + } + + // shrink one frame outGrad + MatrixPtr oneGradTmp = Matrix::create( + expandInput_->getData(), subK * groups_[inpIdx], subN, false, useGpu_); + MatrixPtr vTmp = Matrix::create( + tgtGradData, 1, + imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel, false, + useGpu_); + vTmp->convShrink(*oneGradTmp, imgSizeH_[inpIdx], imgSizeW_[inpIdx], + channel, filterSize_[inpIdx], + filterSize_[inpIdx], stride_[inpIdx], stride_[inpIdx], + padding_[inpIdx], padding_[inpIdx], + outputH_[inpIdx], outputW_[inpIdx], 1.0f, 1.0f); + vTmp->clear(); + oneGradTmp->clear(); + + // move the data-pointer + tgtGradData += imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel; + } +} + +void ExpandConvBaseLayer::bpropWeights(MatrixPtr image, MatrixPtr out, + int inpIdx) { + MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); + + int subM = subM_[inpIdx]; + int subN = subN_[inpIdx]; + int subK = subK_[inpIdx]; + size_t batchSize = image->getHeight(); + resetExpandInput(subK * groups_[inpIdx], subN); + + real *gradData = out->getData(); + + for (size_t n = 0; n < batchSize; n++) { // frame by frame + // expand + expandOneFrame(image, n, inpIdx); + real *wGradData = weightGrad->getData(); + real *expandInData = expandInput_->getData(); + + // expand-mul one-group by one + for (int g = 0; g < groups_[inpIdx]; g++) { + MatrixPtr A = Matrix::create(expandInData, subK, subN, false, useGpu_); + MatrixPtr B = Matrix::create(gradData, subM, subN, true, useGpu_); + MatrixPtr C = Matrix::create(wGradData, subK, subM, false, useGpu_); + C->mul(A, B, 1, 1); + + A->clear(); + B->clear(); + C->clear(); + gradData += subM * subN; + wGradData += subK * subM; + expandInData += subK * subN; + } + } +} + +void ExpandConvBaseLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { + size_t mapW = getOutputSize() / numFilters_; + size_t mapH = v->getElementCnt() / mapW; + MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); + + Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); + + vTmp->transpose(transOutValue_, false); // false means no memory allocation + transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, + numFilters_); + biases->collectBias(*transOutValue_, 1.0f); +} + +void ExpandConvBaseLayer::bpropBiases(MatrixPtr v) { + MatrixPtr biases = + Matrix::create(biases_->getWGrad()->getData(), 1, + biases_->getWGrad()->getElementCnt(), false, useGpu_); + if (sharedBiases_) { + bpropSharedBias(biases, v); + } else { + biases->collectBias(*v, 1.0f); + } + biases->clear(); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvBaseLayer.h b/paddle/gserver/layers/ExpandConvBaseLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..9858fa348c3fc85fdea0c017ca44fa047a6eaf42 --- /dev/null +++ b/paddle/gserver/layers/ExpandConvBaseLayer.h @@ -0,0 +1,85 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "ConvBaseLayer.h" +#include "paddle/math/Matrix.h" +#include + +namespace paddle { + +/** + * @brief A subclass of ConvBaseLayer that is a superclass of both + * ExpandConvLayer and ExpandConvTransLayer + */ +class ExpandConvBaseLayer : public ConvBaseLayer { +protected: + /// For expand convolution. + /// subM_ = numFilters_ / groups_. + IntV subM_; + /// subN_ = outputH_ * outputW_. + IntV subN_; + /// subK_ = channels_ * filterPixels_ * groups_. + IntV subK_; + + /*The expandInput_ and transOutValue_ are used for CPU expand conv calc + * Expand one sample at a time. shape: + * (numChannels * filterPixels_, outputSizeH * outputSizeW) + * */ + MatrixPtr expandInput_; + /// The transpose of output, which is an auxiliary matrix. + MatrixPtr transOutValue_; + +public: + explicit ExpandConvBaseLayer(const LayerConfig& config) + : ConvBaseLayer(config) {} + + ~ExpandConvBaseLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + size_t getOutputSize(); + /** + * Create or resize expandInput_. + */ + void resetExpandInput(size_t height, size_t width); + + /** + * Add shared bias. + */ + void addSharedBias(); + + /** + * Add unshared bias. + */ + void addUnsharedBias(); + /** + * Expand one input sample. + */ + void expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx); + + /** + * Expand one input sample and perform matrix multiplication. + */ + void expandFwdOnce(MatrixPtr image, MatrixPtr out, int inIdx, int startIdx); + + void bpropSharedBias(MatrixPtr biases, MatrixPtr v); + void bpropBiases(MatrixPtr v); + void bpropWeights(MatrixPtr image, MatrixPtr out, int inpIdx); + void bpropActs(MatrixPtr image, MatrixPtr out, int inpIdx); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvLayer.cpp b/paddle/gserver/layers/ExpandConvLayer.cpp index 80a6a62b5c0de768f9cc534adf68405a883ec10f..5ea1fdece5f7b83c7e1d576e7f02a4a2545f0cd8 100644 --- a/paddle/gserver/layers/ExpandConvLayer.cpp +++ b/paddle/gserver/layers/ExpandConvLayer.cpp @@ -24,150 +24,29 @@ REGISTER_LAYER(exconv, ExpandConvLayer); bool ExpandConvLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { /* Initialize the basic convolutional parent class */ - ConvBaseLayer::init(layerMap, parameterMap); - - /* Initialize the projection */ - for (auto &inputConfig : config_.inputs()) { - const ConvConfig &conf = inputConfig.conv_conf(); - subM_.push_back(numFilters_ / conf.groups()); - subN_.push_back(conf.output_x() * conf.output_x()); - subK_.push_back(conf.channels() * conf.filter_size() * conf.filter_size() / - conf.groups()); - /* Consistent caffe mode for multiple input */ - caffeMode_ = conf.caffe_mode(); - } - - /* initialize the weightList */ - CHECK(inputLayers_.size() == parameters_.size()); - for (size_t i = 0; i < inputLayers_.size(); i++) { - size_t height, width; - height = filterPixels_[i] * filterChannels_[i]; - width = numFilters_; - - // create a new weight - CHECK_EQ(parameters_[i]->getSize(), width * height); - Weight* w = new Weight(height, width, parameters_[i]); - weights_.emplace_back(w); - } - + ExpandConvBaseLayer::init(layerMap, parameterMap); return true; } -size_t ExpandConvLayer::getOutputSize() { - CHECK_NE(inputLayers_.size(), 0UL); - size_t layerSize = ConvBaseLayer::calOutputSize(); - subN_.clear(); - for (size_t i = 0; i < inputLayers_.size(); i++) { - subN_.push_back(outputH_[i] * outputW_[i]); - } - return layerSize; -} - -void ExpandConvLayer::resetExpandInput(size_t height, size_t width) { - Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); -} - -void ExpandConvLayer::resetConvOutput(size_t batchSize, int inIdx) { - Matrix::resizeOrCreate(transOutValue_, batchSize * numFilters_, subN_[inIdx], - false, useGpu_); -} - -void ExpandConvLayer::expandOneFrame(MatrixPtr image, size_t startIdx, - int inIdx) { - resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); - real *imgData = image->getData() + startIdx * image->getWidth(); - MatrixPtr imageTmp = Matrix::create( - imgData, 1, imgSizeH_[inIdx] * imgSizeW_[inIdx] * channels_[inIdx], false, - useGpu_); - expandInput_->convExpand(*imageTmp, imgSizeH_[inIdx], imgSizeW_[inIdx], - channels_[inIdx], filterSize_[inIdx], - filterSize_[inIdx], stride_[inIdx], stride_[inIdx], - padding_[inIdx], padding_[inIdx], - outputH_[inIdx], outputW_[inIdx]); - imageTmp->clear(); -} - -void ExpandConvLayer::expandFwdOnce(MatrixPtr image, int inIdx, int startIdx) { - int subM = subM_[inIdx]; - int subN = subN_[inIdx]; - int subK = subK_[inIdx]; - - expandOneFrame(image, startIdx, inIdx); - - real *outData = - getOutputValue()->getData() + startIdx * subN * numFilters_; - - real *wgtData = weights_[inIdx]->getW()->getData(); - real *expInData = expandInput_->getData(); - for (int g = 0; g < groups_[inIdx]; ++g) { - MatrixPtr A = - Matrix::create(wgtData, subK, subM, true, useGpu_); // mark transpose - MatrixPtr B = Matrix::create(expInData, subK, subN, false, useGpu_); - MatrixPtr C = Matrix::create(outData, subM, subN, false, useGpu_); - C->mul(A, B, 1, 1); - - A->clear(); - B->clear(); - C->clear(); - wgtData += subK * subM; - expInData += subK * subN; - outData += subM * subN; - } -} - -void ExpandConvLayer::addSharedBias() { - size_t mapW = getOutputValue()->getWidth() / numFilters_; - size_t mapH = getOutputValue()->getElementCnt() / mapW; - MatrixPtr out = - Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); - - Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); - - out->transpose(transOutValue_, false); // false means no memory allocation - transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, - numFilters_); - - MatrixPtr bias = - Matrix::create(biases_->getW()->getData(), 1, - biases_->getW()->getElementCnt(), false, useGpu_); - transOutValue_->addBias(*bias, 1.0f); - - transOutValue_->reshape(mapW, mapH); - transOutValue_->transpose(out, false); // false means no memory allocation - - out->clear(); - bias->clear(); -} - -void ExpandConvLayer::addUnsharedBias() { - MatrixPtr outValue = getOutputValue(); - MatrixPtr bias = - Matrix::create(biases_->getW()->getData(), 1, - biases_->getW()->getElementCnt(), false, useGpu_); - outValue->addBias(*bias, 1.0f); -} - void ExpandConvLayer::forward(PassType passType) { Layer::forward(passType); /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one colum, and the - * transOutValue correspond sample to one row */ - int batchSize = inputLayers_[0]->getOutputValue()->getWidth(); - batchSize = inputLayers_[0]->getOutputValue()->getHeight(); + int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); resetOutput(batchSize, getOutputSize()); MatrixPtr image = nullptr; - for (size_t i = 0; i != inputLayers_.size(); ++i) { + MatrixPtr outV = getOutputValue(); + for (size_t i = 0; i < inputLayers_.size(); ++i) { LayerPtr prevLayer = getPrev(i); image = prevLayer->getOutputValue(); for (size_t off = 0; off < image->getHeight(); off++) { REGISTER_TIMER_INFO("expandFwdOnce", getName().c_str()); - expandFwdOnce(image, i, off); + expandFwdOnce(image, outV, i, off); } } /* add the bias-vector */ - if (biases_.get() != NULL) { + if (biases_.get()) { if (sharedBiases_) { addSharedBias(); } else { @@ -179,29 +58,6 @@ void ExpandConvLayer::forward(PassType passType) { forwardActivation(); } -void ExpandConvLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { - size_t mapW = v->getWidth() / numFilters_; - size_t mapH = v->getElementCnt() / mapW; - MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); - - Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); - - vTmp->transpose(transOutValue_, false); // false means no memory allocation - vTmp->reshape(transOutValue_->getElementCnt() / numFilters_, numFilters_); - biases->collectBias(*vTmp, 1.0f); -} - -void ExpandConvLayer::bpropBiases(MatrixPtr v) { - MatrixPtr biases = - Matrix::create(biases_->getWGrad()->getData(), 1, - biases_->getWGrad()->getElementCnt(), false, useGpu_); - if (sharedBiases_) { - bpropSharedBias(biases, v); - } else { - biases->collectBias(*v, 1.0f); - } - biases->clear(); -} void ExpandConvLayer::backward(const UpdateCallback &callback) { backwardActivation(); @@ -213,111 +69,18 @@ void ExpandConvLayer::backward(const UpdateCallback &callback) { biases_->getParameterPtr()->incUpdate(callback); } - for (size_t i = 0; i != inputLayers_.size(); ++i) { + for (size_t i = 0; i < inputLayers_.size(); ++i) { /* First, calculate the input layers error */ - bpropActs(outGrad, i); + if (getPrev(i)->getOutputGrad()) { + bpropActs(outGrad, getPrev(i)->getOutputGrad(), i); + } if (weights_[i]->getWGrad()) { /* Then, calculate the W-gradient for the current layer */ - bpropWeights(outGrad, i); + bpropWeights(getPrev(i)->getOutputValue(), outGrad, i); /* Increasing the number of gradient */ weights_[i]->getParameterPtr()->incUpdate(callback); } } } -void ExpandConvLayer::bpropWeights(MatrixPtr v, int inpIdx) { - MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); - MatrixPtr inputV = getPrev(inpIdx)->getOutputValue(); - - int subM = subM_[inpIdx]; - int subN = subN_[inpIdx]; - int subK = subK_[inpIdx]; - size_t batchSize = inputV->getHeight(); - resetExpandInput(subK * groups_[inpIdx], subN); - resetConvOutput(batchSize, inpIdx); - - real *gradData = v->getData(); - - for (size_t n = 0; n < batchSize; n++) { // frame by frame - // expand - expandOneFrame(inputV, n, inpIdx); - real *wGradData = weightGrad->getData(); - real *expandInData = expandInput_->getData(); - - // expand-mul one-group by one - for (int g = 0; g < groups_[inpIdx]; g++) { - MatrixPtr A = Matrix::create(expandInData, subK, subN, false, useGpu_); - MatrixPtr B = Matrix::create(gradData, subM, subN, true, useGpu_); - MatrixPtr C = Matrix::create(wGradData, subK, subM, false, useGpu_); - C->mul(A, B, 1, 1); - - A->clear(); - B->clear(); - C->clear(); - gradData += subM * subN; - wGradData += subK * subM; - expandInData += subK * subN; - } - } -} - -void ExpandConvLayer::bpropActs(MatrixPtr v, int inpIdx) { - LayerPtr prevLayer = getPrev(inpIdx); - if (NULL == prevLayer->getOutputGrad()) { - return; - } - - int subM = subM_[inpIdx]; - int subN = subN_[inpIdx]; - int subK = subK_[inpIdx]; - size_t batchSize = v->getHeight(); - MatrixPtr tgtGrad = prevLayer->getOutputGrad(); - - /* reset the expand-grad memory */ - resetExpandInput(subK * groups_[inpIdx], subN); - resetConvOutput(batchSize, inpIdx); - - real *localGradData = v->getData(); - real *tgtGradData = tgtGrad->getData(); - for (size_t n = 0; n < batchSize; n++) { - real *wgtData = weights_[inpIdx]->getW()->getData(); - real *expandInData = expandInput_->getData(); - - for (int g = 0; g < groups_[inpIdx]; g++) { - // create temporary matrix - MatrixPtr C = Matrix::create(expandInData, subK, subN, false, useGpu_); - MatrixPtr B = Matrix::create(localGradData, subM, subN, false, useGpu_); - MatrixPtr A = Matrix::create(wgtData, subK, subM, false, useGpu_); - C->mul(A, B); // mul - - // clear the temporary matrix - A->clear(); - B->clear(); - C->clear(); - - expandInData += subK * subN; - localGradData += subM * subN; - wgtData += subK * subM; - } - - // shrink one frame outGrad - MatrixPtr oneGradTmp = Matrix::create( - expandInput_->getData(), subK * groups_[inpIdx], subN, false, useGpu_); - MatrixPtr vTmp = Matrix::create( - tgtGradData, 1, - imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channels_[inpIdx], false, - useGpu_); - vTmp->convShrink(*oneGradTmp, imgSizeH_[inpIdx], imgSizeW_[inpIdx], - channels_[inpIdx], filterSize_[inpIdx], - filterSize_[inpIdx], stride_[inpIdx], stride_[inpIdx], - padding_[inpIdx], padding_[inpIdx], - outputH_[inpIdx], outputW_[inpIdx], 1.0f, 1.0f); - vTmp->clear(); - oneGradTmp->clear(); - - // move the data-pointer - tgtGradData += imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channels_[inpIdx]; - } -} - } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvLayer.h b/paddle/gserver/layers/ExpandConvLayer.h index 030a3ba397ff41208bda84d0d6b876359d587c57..c07188a406183416cd57e2d027ba1205f6b65176 100644 --- a/paddle/gserver/layers/ExpandConvLayer.h +++ b/paddle/gserver/layers/ExpandConvLayer.h @@ -15,9 +15,9 @@ limitations under the License. */ #pragma once -#include "ConvBaseLayer.h" #include "paddle/math/Matrix.h" #include +#include "ExpandConvBaseLayer.h" namespace paddle { @@ -28,65 +28,18 @@ namespace paddle { * * The config file api is img_conv_layer. */ -class ExpandConvLayer : public ConvBaseLayer { -protected: - /// For expand convolution. - /// subM_ = numFilters_ / groups_. - IntV subM_; - /// subN_ = outputH_ * outputW_. - IntV subN_; - /// subK_ = channels_ * filterPixels_ * groups_. - IntV subK_; - /// Expand one sample at a time. shape: - /// (numChannels * filterPixels_, outputSizeH * outputSizeW) - MatrixPtr expandInput_; - /// The transpose of output, which is an auxiliary matrix. - MatrixPtr transOutValue_; +class ExpandConvLayer : public ExpandConvBaseLayer { public: - explicit ExpandConvLayer(const LayerConfig& config) : ConvBaseLayer(config) {} + explicit ExpandConvLayer(const LayerConfig& config) : + ExpandConvBaseLayer(config) {} ~ExpandConvLayer() {} bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - size_t getOutputSize(); - - /** - * Create or resize expandInput_. - */ - void resetExpandInput(size_t height, size_t width); - - /** - * Create or resize transOutValue_. - */ - void resetConvOutput(size_t batchSize, int inIdx); - - /** - * Expand one input sample. - */ - void expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx); - - /** - * Expand one input sample and perform matrix multiplication. - */ - void expandFwdOnce(MatrixPtr image, int inIdx, int startIdx); - - /** - * Add shared bias. - */ - void addSharedBias(); - - /** - * Add unshared bias. - */ - void addUnsharedBias(); void forward(PassType passType); - void bpropSharedBias(MatrixPtr biases, MatrixPtr v); - void bpropBiases(MatrixPtr v); void backward(const UpdateCallback& callback); - void bpropWeights(MatrixPtr v, int inpIdx); - void bpropActs(MatrixPtr v, int inpIdx); }; } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a3e160f1f4eb524d39ed90cb17f59f58c690f964 --- /dev/null +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -0,0 +1,92 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "paddle/utils/Logging.h" +#include "paddle/utils/Stat.h" +#include "ExpandConvTransLayer.h" + +/* The implementation of the convTransLayer is basically a swap of forward and + * backward of the original convLayer. + * The variable naming follows the convention of the convLayer. + * */ + +namespace paddle { + +REGISTER_LAYER(exconvt, ExpandConvTransLayer); + +bool ExpandConvTransLayer::init(const LayerMap &layerMap, + const ParameterMap ¶meterMap) { + /* Initialize the basic convolutional parent class */ + ExpandConvBaseLayer::init(layerMap, parameterMap); + + return true; +} + +void ExpandConvTransLayer::forward(PassType passType) { + Layer::forward(passType); + + /* malloc memory for the output_ if necessary */ + int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); + resetOutput(batchSize, getOutputSize()); + + MatrixPtr output = nullptr; + for (size_t i = 0; i < inputLayers_.size(); ++i) { + LayerPtr prevLayer = getPrev(i); + output = prevLayer->getOutputValue(); + REGISTER_TIMER_INFO("shrinkFwd", getName().c_str()); + bpropActs(output, getOutputValue(), i); + } + + /* add the bias-vector */ + if (biases_.get()) { + if (sharedBiases_) { + addSharedBias(); + } else { + addUnsharedBias(); + } + } + + /* activation */ + forwardActivation(); +} + +void ExpandConvTransLayer::backward(const UpdateCallback &callback) { + backwardActivation(); + + MatrixPtr imageGrad = getOutputGrad(); + if (biases_ && biases_->getWGrad()) { + bpropBiases(imageGrad); + /* Increasing the number of gradient */ + biases_->getParameterPtr()->incUpdate(callback); + } + + for (size_t i = 0; i < inputLayers_.size(); ++i) { + /* First, calculate the input layers error */ + for (size_t off = 0; off < imageGrad->getHeight(); off++) { + if (getPrev(i)->getOutputGrad()) { + expandFwdOnce(imageGrad, getPrev(i)->getOutputGrad(), i, off); + } + } + if (weights_[i]->getWGrad()) { + /* Then, calculate the W-gradient for the current layer */ + bpropWeights(imageGrad, getPrev(i)->getOutputValue(), i); + /* Increasing the number of gradient */ + weights_[i]->getParameterPtr()->incUpdate(callback); + } + } +} + + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..87c464a97f2edd5c3528a4434a2aa741d10ddf2e --- /dev/null +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -0,0 +1,44 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "paddle/math/Matrix.h" +#include +#include "ExpandConvBaseLayer.h" + +namespace paddle { + +/** + * @brief A subclass of convolution layer. + * This layer expands input and use matrix multiplication to + * calculate convolution transpose (deconv) operation. + * + * The config file api is img_conv_layer with flag trans=True. + */ +class ExpandConvTransLayer : public ExpandConvBaseLayer { +public: + explicit ExpandConvTransLayer(const LayerConfig& config) : + ExpandConvBaseLayer(config) {} + + ~ExpandConvTransLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + void backward(const UpdateCallback& callback); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/FullMatrixProjection.cpp b/paddle/gserver/layers/FullMatrixProjection.cpp index 8241cbd37ec623622f19ff2ba35c21a4e3e3533a..f17c1b05bd892c7d933e4910887f977ac5cda79b 100644 --- a/paddle/gserver/layers/FullMatrixProjection.cpp +++ b/paddle/gserver/layers/FullMatrixProjection.cpp @@ -52,7 +52,9 @@ void FullMatrixProjection::backward(const UpdateCallback& callback) { } hl_set_sync_flag(syncFlag); - parameter_->incUpdate(callback); + if (weight_->getWGrad()) { + parameter_->incUpdate(callback); + } } } // namespace paddle diff --git a/paddle/gserver/layers/FullyConnectedLayer.h b/paddle/gserver/layers/FullyConnectedLayer.h index 24b6c547e7bc8a60d9374a55074416ea1b9bbc72..334eb4b722f4ff9a794a3818a1cf3087da27692f 100644 --- a/paddle/gserver/layers/FullyConnectedLayer.h +++ b/paddle/gserver/layers/FullyConnectedLayer.h @@ -48,4 +48,3 @@ public: }; } // namespace paddle - diff --git a/paddle/gserver/layers/PoolLayer.cpp b/paddle/gserver/layers/PoolLayer.cpp index 7fc27ac0bd8e05246d87bac0e9692d8496f6601f..2fbc9001f11613cd987e3815f6f31caa8f9979cf 100644 --- a/paddle/gserver/layers/PoolLayer.cpp +++ b/paddle/gserver/layers/PoolLayer.cpp @@ -52,10 +52,8 @@ bool PoolLayer::init(const LayerMap& layerMap, Layer* PoolLayer::create(const LayerConfig& config) { CHECK_EQ(config.inputs_size(), 1); const std::string& pool = config.inputs(0).pool_conf().pool_type(); - if (pool == "max-projection") { - return new MaxPoolProjectionLayer(config); - } else if (pool == "avg-projection") { - return new AvgPoolProjectionLayer(config); + if (pool == "max-projection" || pool == "avg-projection") { + return new PoolProjectionLayer(config); #ifndef PADDLE_ONLY_CPU } else if (CudnnPoolLayer::typeCheck(pool)) { return new CudnnPoolLayer(config); diff --git a/paddle/gserver/layers/PoolProjection.cpp b/paddle/gserver/layers/PoolProjection.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9be5aba3d57d23e462c9ea3608491606f988c35f --- /dev/null +++ b/paddle/gserver/layers/PoolProjection.cpp @@ -0,0 +1,123 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "PoolProjection.h" + +namespace paddle { + +REGISTER_PROJECTION_CREATE_FUNC(pool, &PoolProjection::create); + +PoolProjection::PoolProjection(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu) + : Projection(config, parameter, useGpu) { + const PoolConfig& conf = config_.pool_conf(); + poolType_ = conf.pool_type(); + channels_ = conf.channels(); + sizeX_ = conf.size_x(); + stride_ = conf.stride(); + outputX_ = conf.output_x(); + imgSize_ = conf.img_size(); + confPadding_ = conf.padding(); + + sizeY_ = conf.has_size_y() ? conf.size_y() : conf.size_x(); + imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); + strideY_ = conf.has_stride_y() ? conf.stride_y() : conf.stride(); + confPaddingY_ = conf.has_padding_y() ? conf.padding_y() : conf.padding(); + outputY_ = conf.has_output_y() ? conf.output_y() : conf.output_x(); +} + +size_t PoolProjection::getSize() { + imgSizeY_ = in_->getFrameHeight(); + imgSize_ = in_->getFrameWidth(); + const PoolConfig& conf = config_.pool_conf(); + if (imgSizeY_ == 0) { + imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); + } + if (imgSize_ == 0) { + imgSize_ = conf.img_size(); + } + outputY_ = outputSize(imgSizeY_, sizeY_, confPaddingY_, strideY_, + /* caffeMode */ false); + outputX_ = outputSize(imgSize_, sizeX_, confPadding_, stride_, + /* caffeMode */ false); + + const_cast(out_)->setFrameHeight(outputY_); + const_cast(out_)->setFrameWidth(outputX_); + + return outputY_ * outputX_ * channels_; +} + +PoolProjection* PoolProjection::create(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu) { + const std::string& pool = config.pool_conf().pool_type(); + if (pool == "max-projection") { + return new MaxPoolProjection(config, parameter, useGpu); + } else if (pool == "avg-projection") { + return new AvgPoolProjection(config, parameter, useGpu); + } else { + LOG(FATAL) << "Unknown pool type: " << pool; + return nullptr; + } +} + +void MaxPoolProjection::forward() { + size_t width = getSize(); + CHECK_EQ(width, out_->value->getWidth()); + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + outV->maxPoolForward(*inputV, imgSizeY_, imgSize_, channels_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, confPaddingY_, + confPadding_); +} + +void MaxPoolProjection::backward(const UpdateCallback& callback) { + (void)callback; + MatrixPtr outGrad = out_->grad; + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + MatrixPtr inputGrad = in_->grad; + + if (NULL == inputGrad) { + return; + } + inputGrad->maxPoolBackward(*inputV, imgSizeY_, imgSize_, *outGrad, *outV, + sizeX_, sizeY_, strideY_, stride_, outputY_, + outputX_, 1, 1, confPaddingY_, confPadding_); +} + +void AvgPoolProjection::forward() { + size_t width = getSize(); + CHECK_EQ(width, out_->value->getWidth()); + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + outV->avgPoolForward(*inputV, imgSizeY_, imgSize_, channels_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, confPaddingY_, + confPadding_); +} + +void AvgPoolProjection::backward(const UpdateCallback& callback) { + (void)callback; + + MatrixPtr outputGrad = out_->grad; + MatrixPtr inputGrad = in_->grad; + + if (NULL == inputGrad) { + return; + } + + inputGrad->avgPoolBackward(*outputGrad, imgSizeY_, imgSize_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, 1, 1, + confPaddingY_, confPadding_); +} +} // namespace paddle diff --git a/paddle/gserver/layers/PoolProjection.h b/paddle/gserver/layers/PoolProjection.h new file mode 100644 index 0000000000000000000000000000000000000000..a11e25b729cb7afabdb3547326f269e54ddf42da --- /dev/null +++ b/paddle/gserver/layers/PoolProjection.h @@ -0,0 +1,63 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Projection.h" +#include "paddle/math/MathUtils.h" + +namespace paddle { + +class PoolProjection : public Projection { +protected: + size_t imgSizeY_, imgSize_; + size_t outputY_, outputX_; + size_t strideY_, stride_; + size_t sizeY_, sizeX_; + int confPaddingY_, confPadding_; + size_t channels_; + std::string poolType_; + +public: + PoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu); + + static PoolProjection* create(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu); + + const std::string& getPoolType() const { return poolType_; } + + size_t getSize(); +}; + +class MaxPoolProjection : public PoolProjection { +public: + MaxPoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu) + : PoolProjection(config, parameter, useGpu) {} + + virtual void forward(); + virtual void backward(const UpdateCallback& callback = nullptr); +}; + +class AvgPoolProjection : public PoolProjection { +public: + AvgPoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu) + : PoolProjection(config, parameter, useGpu) {} + + virtual void forward(); + virtual void backward(const UpdateCallback& callback = nullptr); +}; +} // namespace paddle diff --git a/paddle/gserver/layers/PoolProjectionLayer.cpp b/paddle/gserver/layers/PoolProjectionLayer.cpp index 9e8ce778501bbc1f91bfad6d3ab7eb5b1b6f4c80..cabb346d6c99178f7c8ce049d495785c0a488173 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.cpp +++ b/paddle/gserver/layers/PoolProjectionLayer.cpp @@ -18,6 +18,7 @@ limitations under the License. */ namespace paddle { + size_t PoolProjectionLayer::getSize() { CHECK_EQ(inputLayers_.size(), 1UL); size_t layerSize = 0; @@ -37,74 +38,23 @@ size_t PoolProjectionLayer::getSize() { layerSize = outputH_ * outputW_ * channels_; - getOutput().setFrameHeight(outputH_); - getOutput().setFrameWidth(outputW_); return layerSize; } -void MaxPoolProjectionLayer::forward(PassType passType) { - Layer::forward(passType); - - /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one ROW */ - MatrixPtr input = getInputValue(0); - int batchSize = input->getHeight(); - int size = getSize(); - resetOutput(batchSize, size); - - MatrixPtr outV = getOutputValue(); - - outV->maxPoolForward(*input, imgSizeH_, imgSizeW_, channels_, sizeX_, sizeY_, - strideY_, stride_, outputH_, outputW_, confPaddingY_, - confPadding_); -} - -void MaxPoolProjectionLayer::backward(const UpdateCallback& callback) { - (void)callback; - - if (NULL == getInputGrad(0)) { - return; - } - - /* Do derivation */ - MatrixPtr outGrad = getOutputGrad(); - MatrixPtr inputV = getInputValue(0); - MatrixPtr outV = getOutputValue(); - MatrixPtr inputGrad = getInputGrad(0); - - inputGrad->maxPoolBackward(*inputV, imgSizeH_, imgSizeW_, *outGrad, *outV, - sizeX_, sizeY_, strideY_, stride_, outputH_, - outputW_, 1, 1, confPaddingY_, confPadding_); -} - -void AvgPoolProjectionLayer::forward(PassType passType) { +void PoolProjectionLayer::forward(PassType passType) { Layer::forward(passType); - - /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one ROW */ - MatrixPtr input = getInputValue(0); - int batchSize = input->getHeight(); + const Argument& in = getInput(0); + int batchSize = in.value->getHeight(); int size = getSize(); resetOutput(batchSize, size); - - MatrixPtr outV = getOutputValue(); - - outV->avgPoolForward(*input, imgSizeH_, imgSizeW_, channels_, sizeX_, sizeY_, - strideY_, stride_, outputH_, outputW_, confPaddingY_, - confPadding_); + poolProjection_->forward(&in, &output_, passType); } -void AvgPoolProjectionLayer::backward(const UpdateCallback& callback) { +void PoolProjectionLayer::backward(const UpdateCallback& callback) { (void)callback; - if (NULL == getInputGrad(0)) { return; } - /* Do derivation */ - MatrixPtr outputGrad = getOutputGrad(); - MatrixPtr inputGrad = getInputGrad(0); - inputGrad->avgPoolBackward(*outputGrad, imgSizeH_, imgSizeW_, sizeX_, sizeY_, - strideY_, stride_, outputH_, outputW_, 1, 1, - confPaddingY_, confPadding_); + poolProjection_->backward(callback); } } // namespace paddle diff --git a/paddle/gserver/layers/PoolProjectionLayer.h b/paddle/gserver/layers/PoolProjectionLayer.h index 42bbc83c62246dfc8e69aa0b427b27819a701eb6..777b6f39e7cc4ebaa7078ce3378b2688363245e8 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.h +++ b/paddle/gserver/layers/PoolProjectionLayer.h @@ -12,12 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once +#include #include "PoolLayer.h" +#include "PoolProjection.h" #include "paddle/math/Matrix.h" -#include namespace paddle { /** @@ -27,33 +27,18 @@ class PoolProjectionLayer : public PoolLayer { protected: size_t imgSizeH_, imgSizeW_; size_t outputH_, outputW_; + std::unique_ptr poolProjection_; + ProjectionConfig projectionConfig_; public: - size_t getSize(); - explicit PoolProjectionLayer(const LayerConfig& config) : PoolLayer(config) {} -}; -/** - * @brief A layer for max pooling - */ -class MaxPoolProjectionLayer : public PoolProjectionLayer { -public: - explicit MaxPoolProjectionLayer(const LayerConfig& config) - : PoolProjectionLayer(config) {} - - ~MaxPoolProjectionLayer() {} + explicit PoolProjectionLayer(const LayerConfig& config) : PoolLayer(config) { + PoolConfig* conf = projectionConfig_.mutable_pool_conf(); + *conf = config_.inputs(0).pool_conf(); + poolProjection_.reset( + PoolProjection::create(projectionConfig_, nullptr, useGpu_)); + } - virtual void forward(PassType passType); - virtual void backward(const UpdateCallback& callback = nullptr); -}; -/** - * @brief A layer for average pooling - */ -class AvgPoolProjectionLayer : public PoolProjectionLayer { -public: - explicit AvgPoolProjectionLayer(const LayerConfig& config) - : PoolProjectionLayer(config) {} - - ~AvgPoolProjectionLayer() {} + size_t getSize(); virtual void forward(PassType passType); virtual void backward(const UpdateCallback& callback = nullptr); diff --git a/paddle/gserver/layers/Projection.h b/paddle/gserver/layers/Projection.h index 3fa3a0cc230ac4c8616abe0eb2c8ac41bde52d53..203edc5396a53cf72dcad6308335ba4731ba49bc 100644 --- a/paddle/gserver/layers/Projection.h +++ b/paddle/gserver/layers/Projection.h @@ -12,12 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once -#include "paddle/parameter/Parameter.h" -#include "ModelConfig.pb.h" #include "Layer.h" +#include "ModelConfig.pb.h" +#include "paddle/parameter/Parameter.h" namespace paddle { @@ -28,6 +27,11 @@ namespace paddle { Projection::registrar_.registerClass<__class_name>(#__type_name); \ }) +#define REGISTER_PROJECTION_CREATE_FUNC(__type_name, createFunction) \ + static InitFunction __reg_type_##__type_name([]() { \ + Projection::registrar_.registerClass(#__type_name, createFunction); \ + }) + /** * A projection takes one Argument as input, calculate the result and add it * to output Argument. @@ -50,7 +54,8 @@ public: registrar_; /** - * Forward propagation. If backward() will be called, in and out must be kept valid until then. + * Forward propagation. If backward() will be called, in and out must be kept + * valid until then. * @param in input of projection * @param out output of projection * @param passType PASS_TRAIN of PASS_TEST diff --git a/paddle/gserver/layers/ScalingProjection.cpp b/paddle/gserver/layers/ScalingProjection.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c0a7072c6a7cc1d37723f43d1068483779f56437 --- /dev/null +++ b/paddle/gserver/layers/ScalingProjection.cpp @@ -0,0 +1,53 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "Projection.h" + +namespace paddle { + +class ScalingProjection : public Projection { +public: + ScalingProjection(const ProjectionConfig& config, + const ParameterPtr& parameter, bool useGpu) + : Projection(config, parameter, useGpu) { + CHECK_EQ(parameter->getSize(), 1UL); + weight_.reset(new Weight(1, 1, parameter)); + } + + void forward() { + CHECK(in_->value); + out_->value->add(*in_->value, weight_->getW()->getElement(0, 0)); + } + + void backward(const UpdateCallback& callback) { + if (weight_->getWGrad()) { + auto sum = Matrix::create(in_->value->getHeight(), 1, false, useGpu_); + sum->sumOfProducts(*in_->value, *out_->grad, + /* scaleSum= */1, /* scaleDest= */0); + weight_->getWGrad()->sumCols(*sum, + /* scaleSum= */1, /* scaleDest= */1); + parameter_->incUpdate(callback); + } + if (in_->grad) { + in_->grad->add(*out_->grad, weight_->getW()->getElement(0, 0)); + } + } + +protected: + std::unique_ptr weight_; +}; + +REGISTER_PROJECTION(scaling, ScalingProjection); + +} // namespace paddle diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2fcfc8e1ae68a47822ce8f375fb94ecdb196dea6 --- /dev/null +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp @@ -0,0 +1,132 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "SpatialPyramidPoolLayer.h" + +namespace paddle { + +REGISTER_LAYER(spp, SpatialPyramidPoolLayer); + +ProjectionConfig SpatialPyramidPoolLayer::getConfig(size_t imgSizeW, + size_t imgSizeH, + size_t channels, + size_t pyramidLevel, + std::string& poolType) { + ProjectionConfig config; + config.set_type("pool"); + PoolConfig* conf = config.mutable_pool_conf(); + conf->set_channels(channels); + conf->set_img_size(imgSizeW); + conf->set_img_size_y(imgSizeH); + conf->set_pool_type(poolType); + + int numBins = std::pow(2, pyramidLevel); + + int sizeH = std::ceil(imgSizeH / static_cast(numBins)); + int paddingH = (sizeH * numBins - imgSizeH + 1) / 2; + int outSizeH = outputSize(imgSizeH, sizeH, paddingH, sizeH, true); + + int sizeW = std::ceil(imgSizeW / static_cast(numBins)); + int paddingW = (sizeW * numBins - imgSizeW + 1) / 2; + int outSizeW = outputSize(imgSizeW, sizeW, paddingW, sizeW, true); + + conf->set_stride(sizeW); + conf->set_stride_y(sizeH); + conf->set_size_x(sizeW); + conf->set_size_y(sizeH); + conf->set_padding(paddingW); + conf->set_padding_y(paddingH); + conf->set_output_x(outSizeW); + conf->set_output_y(outSizeH); + config.set_output_size(outSizeH * outSizeW * channels); + return config; +} + +size_t SpatialPyramidPoolLayer::getSize() { + CHECK_EQ(inputLayers_.size(), 1UL); + size_t layerSize = 0; + const SppConfig& sppConf = config_.inputs(0).spp_conf(); + imgSizeH_ = inputLayers_[0]->getOutput().getFrameHeight(); + imgSizeW_ = inputLayers_[0]->getOutput().getFrameWidth(); + if (imgSizeH_ == 0) { + imgSizeH_ = sppConf.has_img_size_y() ? sppConf.img_size_y() : imgSizeW_; + } + if (imgSizeW_ == 0) { + imgSizeW_ = sppConf.img_size(); + } + + size_t outputH = 1; + size_t outputW = (std::pow(4, pyramidHeight_) - 1) / (4 - 1); + + layerSize = outputH * outputW * channels_; + return layerSize; +} + +bool SpatialPyramidPoolLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + CHECK_EQ(config_.inputs_size(), 1); + + const SppConfig& sppConf = config_.inputs(0).spp_conf(); + pyramidHeight_ = sppConf.pyramid_height(); + poolType_ = sppConf.pool_type(); + + channels_ = sppConf.channels(); + imgSizeW_ = sppConf.img_size(); + imgSizeH_ = sppConf.has_img_size_y() ? sppConf.img_size_y() : imgSizeW_; + poolProjections_.reserve(pyramidHeight_); + projCol_.reserve(pyramidHeight_); + projOutput_.resize(pyramidHeight_); + + size_t startCol = 0; + size_t endCol = 0; + for (size_t i = 0; i < pyramidHeight_; i++) { + poolProjections_.emplace_back(PoolProjection::create( + getConfig(imgSizeW_, imgSizeH_, channels_, i, poolType_), nullptr, + useGpu_)); + endCol += poolProjections_[i]->getOutputSize(); + projCol_.push_back(std::make_pair(startCol, endCol)); + startCol = endCol; + } + CHECK_EQ(endCol, getSize()); + return true; +} + +void SpatialPyramidPoolLayer::forward(PassType passType) { + Layer::forward(passType); + + int batchSize = getInput(0).getBatchSize(); + resetOutput(batchSize, getSize()); + for (size_t i = 0; i < pyramidHeight_; i++) { + size_t startCol = projCol_[i].first; + size_t endCol = projCol_[i].second; + projOutput_[i].value = output_.value->subColMatrix(startCol, endCol); + if (output_.grad) { + projOutput_[i].grad = output_.grad->subColMatrix(startCol, endCol); + } + } + for (size_t i = 0; i < pyramidHeight_; i++) { + poolProjections_[i]->forward(&getInput(0), &projOutput_[i], passType); + } +} + +void SpatialPyramidPoolLayer::backward(const UpdateCallback& callback) { + for (size_t i = 0; i < pyramidHeight_; i++) { + if (poolProjections_[i]) { + poolProjections_[i]->backward(callback); + } + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.h b/paddle/gserver/layers/SpatialPyramidPoolLayer.h new file mode 100644 index 0000000000000000000000000000000000000000..e15b6d2f85c6f5b9620e28aaef9c6246341611f9 --- /dev/null +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" +#include "PoolProjection.h" +#include "paddle/math/MathUtils.h" +#include "paddle/utils/Logging.h" + +namespace paddle { +/** + * @brief A layer for spatial pyramid pooling on the input image by taking + * the max, average, etc. within regions, so that the result vector of + * different sized images are of the same size. + * + * The config file api is spp_layer. + */ + +class SpatialPyramidPoolLayer : public Layer { +protected: + size_t channels_; + size_t imgSizeW_; + size_t imgSizeH_; + size_t pyramidHeight_; + std::string poolType_; + + std::vector> poolProjections_; + std::vector projOutput_; + std::vector> projCol_; + +public: + explicit SpatialPyramidPoolLayer(const LayerConfig& config) : Layer(config) {} + + ~SpatialPyramidPoolLayer() {} + + virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + ProjectionConfig getConfig(size_t sizeX_, size_t sizeY_, size_t channels, + size_t pyamidLevel_, std::string& poolType_); + size_t getSize(); + + virtual void forward(PassType passType); + virtual void backward(const UpdateCallback& callback = nullptr); +}; +} // namespace paddle diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index 26ee2b3aae64abfce69b543f13ab0f4254757fd8..0651d0b4733ea9c3f54a42169774217b65091aa6 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -26,6 +26,14 @@ add_unittest_without_exec(test_ActivationGrad TestUtil.cpp) add_test(NAME test_ActivationGrad COMMAND test_ActivationGrad) +################# test_ConvTrans ####################### +add_unittest_without_exec(test_ConvTrans + test_ConvTrans.cpp + LayerGradUtil.cpp + TestUtil.cpp) + +add_test(NAME test_ConvTrans + COMMAND test_ConvTrans) ################## test_Evaluator ####################### add_unittest(test_Evaluator diff --git a/paddle/gserver/tests/__init__.py b/paddle/gserver/tests/__init__.py index 7f9e87eee6037666b86420fba194624859d356b3..c90af2ee000d46a032984ee23559e7e99b49ddad 100644 --- a/paddle/gserver/tests/__init__.py +++ b/paddle/gserver/tests/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/gserver/tests/pyDataProvider.py b/paddle/gserver/tests/pyDataProvider.py index c3155e7adea04d472cca2ed74ec31c22bc2d1586..91863b4175b1a58cb7d475732f293f32a3a6ed5a 100644 --- a/paddle/gserver/tests/pyDataProvider.py +++ b/paddle/gserver/tests/pyDataProvider.py @@ -16,72 +16,79 @@ import numpy import struct import traceback + def header_creator(): ret = "" - ret += struct.pack('i', 3) # slot num - ret += struct.pack('i', 1) # sequence flag - ret += struct.pack('i', 0) # slot0 dense type - ret += struct.pack('i', 3) # slot0 dim - ret += struct.pack('i', 1) # slot1 sparse non value type - ret += struct.pack('i', 7) # slot1 dim - ret += struct.pack('i', 3) # slot2 index type - ret += struct.pack('i', 2) # slot2 dim + ret += struct.pack('i', 3) # slot num + ret += struct.pack('i', 1) # sequence flag + ret += struct.pack('i', 0) # slot0 dense type + ret += struct.pack('i', 3) # slot0 dim + ret += struct.pack('i', 1) # slot1 sparse non value type + ret += struct.pack('i', 7) # slot1 dim + ret += struct.pack('i', 3) # slot2 index type + ret += struct.pack('i', 2) # slot2 dim return ret + def dense_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot0 sample num - for i in range(sample_num): # slot0 value + ret += struct.pack('i', sample_num) # slot0 sample num + for i in range(sample_num): # slot0 value ret += struct.pack('f', 1.0) ret += struct.pack('f', 2.0) ret += struct.pack('f', 3.0) return ret + def sparse_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot1 sample num - for i in range(sample_num): # slot1 index + ret += struct.pack('i', sample_num) # slot1 sample num + for i in range(sample_num): # slot1 index ret += struct.pack('i', i * 2) - ret += struct.pack('i', sample_num * 2) #slot1 length - for i in range(sample_num): # slot1 value + ret += struct.pack('i', sample_num * 2) #slot1 length + for i in range(sample_num): # slot1 value ret += struct.pack('i', 1) ret += struct.pack('i', 2) return ret + def index_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot2 sample num - for i in range(sample_num): # slot2 value + ret += struct.pack('i', sample_num) # slot2 sample num + for i in range(sample_num): # slot2 value ret += struct.pack('i', 0) return ret + def sequenceStartPositions_creator(): ret = "" - ret += struct.pack('i', 2) # slot0 sequence num - ret += struct.pack('i', 0) # slot0 sequence value1 - ret += struct.pack('i', 1) # slot0 sequence value2 - ret += struct.pack('i', 1) # slot1 sequence num - ret += struct.pack('i', 0) # slot1 sequence value1 - ret += struct.pack('i', 2) # slot2 sequence num - ret += struct.pack('i', 0) # slot2 sequence value1 - ret += struct.pack('i', 1) # slot2 sequence value2 + ret += struct.pack('i', 2) # slot0 sequence num + ret += struct.pack('i', 0) # slot0 sequence value1 + ret += struct.pack('i', 1) # slot0 sequence value2 + ret += struct.pack('i', 1) # slot1 sequence num + ret += struct.pack('i', 0) # slot1 sequence value1 + ret += struct.pack('i', 2) # slot2 sequence num + ret += struct.pack('i', 0) # slot2 sequence value1 + ret += struct.pack('i', 1) # slot2 sequence value2 return ret + def subSequenceStartPositions_creator(): ret = "" - ret += struct.pack('i', 3) # slot0 subsequence num - ret += struct.pack('i', 0) # slot0 subsequence value1 - ret += struct.pack('i', 1) # slot0 subsequence value2 - ret += struct.pack('i', 2) # slot0 subsequence value3 - ret += struct.pack('i', 2) # slot1 subsequence num - ret += struct.pack('i', 0) # slot1 subsequence value1 - ret += struct.pack('i', 1) # slot1 subsequence value2 - ret += struct.pack('i', 3) # slot2 subsequence num - ret += struct.pack('i', 0) # slot2 subsequence value1 - ret += struct.pack('i', 1) # slot2 subsequence value2 - ret += struct.pack('i', 2) # slot2 subsequence value3 + ret += struct.pack('i', 3) # slot0 subsequence num + ret += struct.pack('i', 0) # slot0 subsequence value1 + ret += struct.pack('i', 1) # slot0 subsequence value2 + ret += struct.pack('i', 2) # slot0 subsequence value3 + ret += struct.pack('i', 2) # slot1 subsequence num + ret += struct.pack('i', 0) # slot1 subsequence value1 + ret += struct.pack('i', 1) # slot1 subsequence value2 + ret += struct.pack('i', 3) # slot2 subsequence num + ret += struct.pack('i', 0) # slot2 subsequence value1 + ret += struct.pack('i', 1) # slot2 subsequence value2 + ret += struct.pack('i', 2) # slot2 subsequence value3 return ret + class SimpleDataProvider: def __init__(self, *file_list): self.file_list = file_list @@ -93,17 +100,18 @@ class SimpleDataProvider: pass def getHeader(self): - return header_creator() + return header_creator() def getNextBatch(self, batch_size): ret = "" - ret += struct.pack('i', 2) # batch size - ret += dense_value_creator(2) # slot0 - ret += sparse_value_creator(2) # slot1 - ret += index_value_creator(2) # slot2 + ret += struct.pack('i', 2) # batch size + ret += dense_value_creator(2) # slot0 + ret += sparse_value_creator(2) # slot1 + ret += index_value_creator(2) # slot2 ret += sequenceStartPositions_creator() return ret + class SimpleNestDataProvider: def __init__(self, *file_list): self.file_list = file_list @@ -119,14 +127,15 @@ class SimpleNestDataProvider: def getNextBatch(self, batch_size): ret = "" - ret += struct.pack('i', 2) # batch size - ret += dense_value_creator(4) # slot0 - ret += sparse_value_creator(4) # slot1 - ret += index_value_creator(4) # slot2 + ret += struct.pack('i', 2) # batch size + ret += dense_value_creator(4) # slot0 + ret += sparse_value_creator(4) # slot1 + ret += index_value_creator(4) # slot2 ret += sequenceStartPositions_creator() ret += subSequenceStartPositions_creator() return ret + if __name__ == "__main__": # test code data_provider = SimpleDataProvider('./test_batch') diff --git a/paddle/gserver/tests/rnn_data_provider.py b/paddle/gserver/tests/rnn_data_provider.py index 321c78cb1741bcfcbd7df2fd83ff6ba5ba910971..715ac08a42d05cec9c7f4b09a0447d44835d417d 100644 --- a/paddle/gserver/tests/rnn_data_provider.py +++ b/paddle/gserver/tests/rnn_data_provider.py @@ -22,18 +22,20 @@ data = [ [[[0, 2], [2, 5], [0, 1, 2]], 1], ] + # Used for sequence_nest_rnn.conf -@provider(input_types=[integer_value_sub_sequence(10), - integer_value(3)], - should_shuffle=False) +@provider( + input_types=[integer_value_sub_sequence(10), integer_value(3)], + should_shuffle=False) def process_subseq(settings, file_name): for d in data: yield d + # Used for sequence_rnn.conf -@provider(input_types=[integer_value_sequence(10), - integer_value(3)], - should_shuffle=False) +@provider( + input_types=[integer_value_sequence(10), integer_value(3)], + should_shuffle=False) def process_seq(settings, file_name): for d in data: seq = [] @@ -41,18 +43,20 @@ def process_seq(settings, file_name): seq += subseq yield seq, d[1] + # Used for sequence_nest_rnn_multi_input.conf -@provider(input_types=[integer_value_sub_sequence(10), - integer_value(3)], - should_shuffle=False) +@provider( + input_types=[integer_value_sub_sequence(10), integer_value(3)], + should_shuffle=False) def process_subseq2(settings, file_name): for d in data: yield d + # Used for sequence_rnn_multi_input.conf -@provider(input_types=[integer_value_sequence(10), - integer_value(3)], - should_shuffle=False) +@provider( + input_types=[integer_value_sequence(10), integer_value(3)], + should_shuffle=False) def process_seq2(settings, file_name): for d in data: seq = [] @@ -60,31 +64,34 @@ def process_seq2(settings, file_name): seq += subseq yield seq, d[1] + ########################################################### data2 = [ - [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]] ,0], - [[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]], 1], + [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]], 0], + [[[0, 2], [2, 5], [0, 1, 2]], [[1, 5], [4], [2, 3, 6, 1]], 1], ] + # Used for sequence_nest_rnn_multi_unequalength_inputs.conf -@provider(input_types=[integer_value_sub_sequence(10), - integer_value_sub_sequence(10), - integer_value(2)], - should_shuffle=False) +@provider( + input_types=[ + integer_value_sub_sequence(10), integer_value_sub_sequence(10), + integer_value(2) + ], + should_shuffle=False) def process_unequalength_subseq(settings, file_name): for d in data2: yield d # Used for sequence_rnn_multi_unequalength_inputs.conf -@provider(input_types=[integer_value_sequence(10), - integer_value_sequence(10), - integer_value(2)], - should_shuffle=False) +@provider( + input_types=[ + integer_value_sequence(10), integer_value_sequence(10), integer_value(2) + ], + should_shuffle=False) def process_unequalength_seq(settings, file_name): for d in data2: - words1=reduce(lambda x,y: x+y, d[0]) - words2=reduce(lambda x,y: x+y, d[1]) + words1 = reduce(lambda x, y: x + y, d[0]) + words2 = reduce(lambda x, y: x + y, d[1]) yield words1, words2, d[2] - - diff --git a/paddle/gserver/tests/sequenceGen.py b/paddle/gserver/tests/sequenceGen.py index b166e778d7a33f444b91d6b37c74352a72f4ac10..fab876fd30da0a80774d06028ae2321e12354d59 100644 --- a/paddle/gserver/tests/sequenceGen.py +++ b/paddle/gserver/tests/sequenceGen.py @@ -20,8 +20,9 @@ from paddle.trainer.PyDataProvider2 import * def hook(settings, dict_file, **kwargs): settings.word_dict = dict_file - settings.input_types = [integer_value_sequence(len(settings.word_dict)), - integer_value(3)] + settings.input_types = [ + integer_value_sequence(len(settings.word_dict)), integer_value(3) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -32,16 +33,19 @@ def process(settings, file_name): label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if - w in settings.word_dict] + word_slot = [ + settings.word_dict[w] for w in words if w in settings.word_dict + ] yield word_slot, label ## for hierarchical sequence network def hook2(settings, dict_file, **kwargs): settings.word_dict = dict_file - settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), - integer_value_sequence(3)] + settings.input_types = [ + integer_value_sub_sequence(len(settings.word_dict)), + integer_value_sequence(3) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -55,8 +59,10 @@ def process2(settings, file_name): label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if - w in settings.word_dict] + word_slot = [ + settings.word_dict[w] for w in words + if w in settings.word_dict + ] label_list.append(label) word_slot_list.append(word_slot) else: diff --git a/paddle/gserver/tests/sequence_layer_group.conf b/paddle/gserver/tests/sequence_layer_group.conf index ac031b31280df297246c1ea2e279fc2c595bd8b7..087aa96ccb5a7fc2b6d4f5ce81de4e820580570a 100644 --- a/paddle/gserver/tests/sequence_layer_group.conf +++ b/paddle/gserver/tests/sequence_layer_group.conf @@ -21,15 +21,16 @@ dict_file = dict() for line_count, line in enumerate(open(dict_path, "r")): dict_file[line.strip()] = line_count -define_py_data_sources2(train_list='gserver/tests/Sequence/train.list', - test_list=None, - module='sequenceGen', - obj='process', - args={"dict_file":dict_file}) +define_py_data_sources2( + train_list='gserver/tests/Sequence/train.list', + test_list=None, + module='sequenceGen', + obj='process', + args={"dict_file": dict_file}) settings(batch_size=5) ######################## network configure ################################ -dict_dim = len(open(dict_path,'r').readlines()) +dict_dim = len(open(dict_path, 'r').readlines()) word_dim = 128 hidden_dim = 256 label_dim = 3 @@ -39,21 +40,24 @@ data = data_layer(name="word", size=dict_dim) emb = embedding_layer(input=data, size=word_dim) # (lstm_input + lstm) is equal to lstmemory -with mixed_layer(size=hidden_dim*4) as lstm_input: +with mixed_layer(size=hidden_dim * 4) as lstm_input: lstm_input += full_matrix_projection(input=emb) -lstm = lstmemory_group(input=lstm_input, - size=hidden_dim, - act=TanhActivation(), - gate_act=SigmoidActivation(), - state_act=TanhActivation(), - lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) +lstm = lstmemory_group( + input=lstm_input, + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) lstm_last = last_seq(input=lstm) -with mixed_layer(size=label_dim, - act=SoftmaxActivation(), - bias_attr=True) as output: +with mixed_layer( + size=label_dim, act=SoftmaxActivation(), bias_attr=True) as output: output += full_matrix_projection(input=lstm_last) -outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +outputs( + classification_cost( + input=output, label=data_layer( + name="label", size=1))) diff --git a/paddle/gserver/tests/sequence_nest_layer_group.conf b/paddle/gserver/tests/sequence_nest_layer_group.conf index 38c60b657b969f9fbcf46a00c542fa100da5a877..93a0f6da7905c0b00cf70296143ded2d4431e430 100644 --- a/paddle/gserver/tests/sequence_nest_layer_group.conf +++ b/paddle/gserver/tests/sequence_nest_layer_group.conf @@ -21,15 +21,16 @@ dict_file = dict() for line_count, line in enumerate(open(dict_path, "r")): dict_file[line.strip()] = line_count -define_py_data_sources2(train_list='gserver/tests/Sequence/train.list.nest', - test_list=None, - module='sequenceGen', - obj='process2', - args={"dict_file":dict_file}) +define_py_data_sources2( + train_list='gserver/tests/Sequence/train.list.nest', + test_list=None, + module='sequenceGen', + obj='process2', + args={"dict_file": dict_file}) settings(batch_size=2) ######################## network configure ################################ -dict_dim = len(open(dict_path,'r').readlines()) +dict_dim = len(open(dict_path, 'r').readlines()) word_dim = 128 hidden_dim = 256 label_dim = 3 @@ -38,37 +39,46 @@ data = data_layer(name="word", size=dict_dim) emb_group = embedding_layer(input=data, size=word_dim) + # (lstm_input + lstm) is equal to lstmemory def lstm_group(lstm_group_input): - with mixed_layer(size=hidden_dim*4) as group_input: - group_input += full_matrix_projection(input=lstm_group_input) + with mixed_layer(size=hidden_dim * 4) as group_input: + group_input += full_matrix_projection(input=lstm_group_input) - lstm_output = lstmemory_group(input=group_input, - name="lstm_group", - size=hidden_dim, - act=TanhActivation(), - gate_act=SigmoidActivation(), - state_act=TanhActivation(), - lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + lstm_output = lstmemory_group( + input=group_input, + name="lstm_group", + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) return lstm_output -lstm_nest_group = recurrent_group(input=SubsequenceInput(emb_group), - step=lstm_group, - name="lstm_nest_group") + +lstm_nest_group = recurrent_group( + input=SubsequenceInput(emb_group), step=lstm_group, name="lstm_nest_group") # hasSubseq ->(seqlastins) seq -lstm_last = last_seq(input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) +lstm_last = last_seq( + input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) # seq ->(expand) hasSubseq -lstm_expand = expand_layer(input=lstm_last, expand_as=emb_group, expand_level=ExpandLevel.FROM_SEQUENCE) +lstm_expand = expand_layer( + input=lstm_last, + expand_as=emb_group, + expand_level=ExpandLevel.FROM_SEQUENCE) # hasSubseq ->(average) seq -lstm_average = pooling_layer(input=lstm_expand, - pooling_type=AvgPooling(), - agg_level=AggregateLevel.EACH_SEQUENCE) +lstm_average = pooling_layer( + input=lstm_expand, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) -with mixed_layer(size=label_dim, - act=SoftmaxActivation(), - bias_attr=True) as output: +with mixed_layer( + size=label_dim, act=SoftmaxActivation(), bias_attr=True) as output: output += full_matrix_projection(input=lstm_average) -outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +outputs( + classification_cost( + input=output, label=data_layer( + name="label", size=1))) diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bff7222b29907cb66d79decea76e1b5e26205ddf --- /dev/null +++ b/paddle/gserver/tests/test_ConvTrans.cpp @@ -0,0 +1,246 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include "paddle/gserver/layers/DataLayer.h" +#include "ModelConfig.pb.h" +#include "paddle/trainer/Trainer.h" +#include "paddle/utils/GlobalConstants.h" +#include "paddle/gserver/layers/ExpandConvTransLayer.h" +#include "paddle/math/MathUtils.h" + +#include "TestUtil.h" +#include "LayerGradUtil.h" + +using namespace paddle; // NOLINT +using namespace std; // NOLINT + +P_DECLARE_bool(use_gpu); +P_DECLARE_int32(gpu_id); +P_DECLARE_double(checkgrad_eps); +P_DECLARE_bool(thread_local_rand_use_global_seed); +P_DECLARE_bool(prev_batch_state); + +// Test that the convTrans forward is the same as conv backward +TEST(Layer, convTransLayerFwd) { + // Setting up conv-trans layer + TestConfig configt; + configt.biasSize = 3; + configt.layerConfig.set_type("exconvt"); + configt.layerConfig.set_num_filters(3); + configt.layerConfig.set_partial_sum(1); + configt.layerConfig.set_shared_biases(true); + + configt.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 384}); + LayerInputConfig* input = configt.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(4); + conv->set_channels(16); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(3 / conv->groups()); + conv->set_img_size(16); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); + configt.layerConfig.set_size(conv->img_size() * conv->img_size() * + configt.layerConfig.num_filters()); + configt.layerConfig.set_name("convTrans"); + + // data layer initialize + std::vector dataLayers; + LayerMap layerMap; + vector datas; + initDataLayer(configt, &dataLayers, &datas, &layerMap, "convTrans", + 100, false, false); + // test layer initialize + std::vector parameters; + LayerPtr convtLayer; + initTestLayer(configt, &layerMap, ¶meters, &convtLayer); + convtLayer->getBiasParameter()->zeroMem(); + convtLayer->forward(PASS_GC); + + // Setting up conv-layer config + TestConfig config; + config.biasSize = 16; + config.layerConfig.set_type("exconv"); + config.layerConfig.set_num_filters(16); + config.layerConfig.set_partial_sum(1); + config.layerConfig.set_shared_biases(true); + + config.inputDefs.push_back({INPUT_DATA, "layer_1", 768, 384}); + input = config.layerConfig.add_inputs(); + conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(4); + conv->set_channels(3); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(conv->channels() / conv->groups()); + conv->set_img_size(16); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); + config.layerConfig.set_size(conv->output_x() * conv->output_x() * + config.layerConfig.num_filters()); + config.layerConfig.set_name("conv"); + + // data layer initialize + std::vector dataLayers2; + LayerMap layerMap2; + vector datas2; + initDataLayer(config, &dataLayers2, &datas2, &layerMap2, "conv", + 100, false, false); + // test layer initialize + std::vector parameters2; + LayerPtr convLayer; + initTestLayer(config, &layerMap2, ¶meters2, &convLayer); + + // Sync convLayer and convtLayer parameter + convLayer->getBiasParameter()->zeroMem(); + convLayer->getParameters()[0]->getBuf(PARAMETER_VALUE)->copyFrom( + *(convtLayer->getParameters()[0]->getBuf(PARAMETER_VALUE))); + + // Set convLayer outputGrad as convTransLayer input value + convLayer->forward(PASS_GC); + convLayer->getOutput().grad->copyFrom(*(dataLayers[0]->getOutputValue())); + + vector callbackFlags(parameters2.size(), 0); + auto callback = [&](Parameter* para) { ++callbackFlags[para->getID()]; }; + convLayer->backward(callback); + + // Check that the convLayer backward is the same as convTransLayer forward + checkMatrixEqual(convtLayer->getOutputValue(), + dataLayers2[0]->getOutputGrad()); +} + + +// Do one forward pass of convTrans layer and check to see if its output +// matches the given result +void doOneConvtTest(size_t imgSize, size_t output_x, size_t stride, + size_t padding, size_t filter_size, MatrixPtr& result) { + TestConfig configt; + configt.biasSize = 1; + configt.layerConfig.set_type("exconvt"); + configt.layerConfig.set_num_filters(1); + configt.layerConfig.set_partial_sum(1); + configt.layerConfig.set_shared_biases(true); + + configt.inputDefs.push_back({INPUT_DATA, "layer_0", output_x * output_x, + filter_size * filter_size}); + LayerInputConfig* input = configt.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(filter_size); + conv->set_filter_size_y(filter_size); + conv->set_channels(1); + conv->set_padding(padding); + conv->set_padding_y(padding); + conv->set_stride(stride); + conv->set_stride_y(stride); + conv->set_groups(1); + conv->set_filter_channels(1); + conv->set_img_size(imgSize); + conv->set_output_x(output_x); + + configt.layerConfig.set_size(conv->img_size() * conv->img_size() * + configt.layerConfig.num_filters()); + configt.layerConfig.set_name("convTrans"); + + std::vector dataLayers; + LayerMap layerMap; + vector datas; + initDataLayer(configt, &dataLayers, &datas, &layerMap, "convTrans", + 1, false, false); + dataLayers[0]->getOutputValue()->zeroMem(); + dataLayers[0]->getOutputValue()->add(1.0); + + // test layer initialize + std::vector parameters; + LayerPtr convtLayer; + initTestLayer(configt, &layerMap, ¶meters, &convtLayer); + convtLayer->getBiasParameter()->zeroMem(); + convtLayer->getParameters()[0]->zeroMem(); + convtLayer->getParameters()[0]->getBuf(PARAMETER_VALUE)->add(1.0); + convtLayer->forward(PASS_GC); + + checkMatrixEqual(convtLayer->getOutputValue(), result); +} + +TEST(Layer, convTransLayerFwd2) { + MatrixPtr result; + result = Matrix::create(1, 5 * 5, false, false); + result->zeroMem(); + result->add(1.0); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 1, + /* stride */ 1, + /* padding */ 0, + /* filter_size */ 5, + result); + + float resultData[] = {1, 2, 2, 2, 1, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 1, 2, 2, 2, 1}; + result->setData(resultData); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 1, + /* padding */ 0, + /* filter_size */ 4, + result); + + float resultData2[] = {1, 2, 2, 2, 1, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 1, 2, 2, 2, 1}; + result->setData(resultData2); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 2, + /* padding */ 1, + /* filter_size */ 5, + result); + + float resultData3[] = {1, 1, 2, 1, 1, + 1, 1, 2, 1, 1, + 2, 2, 4, 2, 2, + 1, 1, 2, 1, 1, + 1, 1, 2, 1, 1}; + result->setData(resultData3); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 2, + /* padding */ 0, + /* filter_size */ 3, + result);} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + FLAGS_thread_local_rand_use_global_seed = true; + srand(1); + return RUN_ALL_TESTS(); +} diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 55a6f66ac51710517345e84a8091788622f3d501..a79dfe39c9bb26c7b2acec1051699e1804494d93 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -13,15 +13,15 @@ See the License for the specific language governing permissions and limitations under the License. */ #include -#include #include -#include "paddle/gserver/layers/DataLayer.h" +#include #include "ModelConfig.pb.h" +#include "paddle/gserver/layers/DataLayer.h" #include "paddle/trainer/Trainer.h" #include "paddle/math/MathUtils.h" -#include "TestUtil.h" #include "LayerGradUtil.h" +#include "TestUtil.h" using namespace paddle; // NOLINT using namespace std; // NOLINT @@ -135,6 +135,17 @@ TEST(Projection, identity) { } } +TEST(Projection, scaling) { + ProjectionConfig conf; + conf.set_type("scaling"); + conf.set_input_size(10); + conf.set_output_size(10); + for (auto useGpu : {false}) { + testProjectionGrad(conf, INPUT_DATA, /* parameterSize */ 1, + /* batchSize */ 100, useGpu); + } +} + #ifndef PADDLE_ONLY_CPU TEST(Projection, conv) { const int NUM_FILTERS = 16; @@ -323,6 +334,8 @@ void testConvLayer(const string& type, bool trans, bool useGpu) { config.layerConfig.num_filters()); testLayerGrad(config, "conv", 100, trans, useGpu); + // Use small batch_size and useWeight=true to test biasGrad + testLayerGrad(config, "conv", 2, trans, useGpu, true, 0.02); } TEST(Layer, convLayer) { @@ -333,6 +346,46 @@ TEST(Layer, convLayer) { #endif } + +void testConvTransLayer(const string& type, bool trans, bool useGpu) { + TestConfig config; + config.biasSize = 3; + config.layerConfig.set_type(type); + config.layerConfig.set_num_filters(3); + config.layerConfig.set_partial_sum(1); + config.layerConfig.set_shared_biases(true); + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 288}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(3); + conv->set_channels(16); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(3 / conv->groups()); + conv->set_img_size(16); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); + + config.layerConfig.set_size(conv->img_size() * conv->img_size() * + config.layerConfig.num_filters()); + + testLayerGrad(config, "convTrans", 100, trans, useGpu); + // Use small batch_size and useWeight=true to test biasGrad + testLayerGrad(config, "convTrans", 2, trans, useGpu, true, 0.02); +} + +TEST(Layer, convTransLayer) { + for (auto useGpu : {false, true}) { + testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ useGpu); + } +} + TEST(Layer, blockExpandLayer) { TestConfig config; config.biasSize = 0; @@ -486,7 +539,7 @@ TEST(Layer, multi_cross) { } } -TEST(Layer, multi_binary_label) { +TEST(Layer, multi_binary_label_sparse_mat) { TestConfig config; config.layerConfig.set_type("multi_binary_label_cross_entropy"); config.biasSize = 0; @@ -496,9 +549,26 @@ TEST(Layer, multi_binary_label) { config.layerConfig.add_inputs(); config.layerConfig.add_inputs(); - // Not support GPU now - testLayerGrad(config, "multi_binary_label_cross_entropy", 100, - /* trans */ false, /* useGpu */ false); + for (auto useGpu : {false, true}) { + testLayerGrad(config, "multi_binary_label_cross_entropy", 100, + /* trans */ false, useGpu); + } +} + +TEST(layer, multi_binary_label_id) { + TestConfig config; + config.layerConfig.set_type("multi_binary_label_cross_entropy"); + config.biasSize = 0; + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); + config.inputDefs.push_back({INPUT_LABEL, "layer_1", 10, 0}); + config.layerConfig.add_inputs(); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "multi_binary_label_cross_entropy", 100, + /* trans */ false, useGpu); + } } TEST(Layer, multi_cross_with_selfnorm) { @@ -939,6 +1009,32 @@ TEST(Layer, PoolLayer) { #endif } +void testSppLayer(const string& poolType, const int pyramidHeight, bool trans, + bool useGpu) { + TestConfig config; + config.layerConfig.set_type("spp"); + config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + SppConfig* sppConfig = input->mutable_spp_conf(); + sppConfig->set_pool_type(poolType); + sppConfig->set_pyramid_height(pyramidHeight); + sppConfig->set_channels(16); + sppConfig->set_img_size(10); + sppConfig->set_img_size_y(20); + int outputSize = (std::pow(4, sppConfig->pyramid_height()) - 1) / (4 - 1); + config.layerConfig.set_size(outputSize * sppConfig->channels()); + testLayerGrad(config, "spp", 100, trans, useGpu); +} + +TEST(Layer, SpatialPyramidPoolLayer) { + for (auto useGpu : {false, true}) { + for (auto pyramidHeight : {1, 2, 3}) { + testSppLayer("avg-projection", pyramidHeight, false, useGpu); + testSppLayer("max-projection", pyramidHeight, false, useGpu); + } + } +} + TEST(Layer, rankCostLayer) { TestConfig config; config.layerConfig.set_type("rank-cost"); @@ -956,6 +1052,19 @@ TEST(Layer, rankCostLayer) { } } +TEST(Layer, sumCostLayer) { + TestConfig config; + config.layerConfig.set_type("sum_cost"); + config.biasSize = 0; + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "sum_cost", 100, false, useGpu); + } +} + TEST(Layer, weightedRankCostLayer) { TestConfig config; config.layerConfig.set_type("rank-cost"); diff --git a/paddle/gserver/tests/test_PyDataProvider2.py b/paddle/gserver/tests/test_PyDataProvider2.py index 71c3335231e52132e6c7e9aaf0cb92d0db2e20df..7ca30198fb1d0e7384db2c28524c7898dcd27e50 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.py +++ b/paddle/gserver/tests/test_PyDataProvider2.py @@ -33,16 +33,19 @@ def test_init_hooker(setting, value, **kwargs): setting.value = value -@provider(input_types=[dense_vector(20, seq_type=SequenceType.NO_SEQUENCE)], - init_hook=test_init_hooker) +@provider( + input_types=[dense_vector( + 20, seq_type=SequenceType.NO_SEQUENCE)], + init_hook=test_init_hooker) def test_init_hook(setting, filename): for i in xrange(200): yield setting.value -@provider( - input_types=[ - sparse_binary_vector(30000, seq_type=SequenceType.NO_SEQUENCE)]) +@provider(input_types=[ + sparse_binary_vector( + 30000, seq_type=SequenceType.NO_SEQUENCE) +]) def test_sparse_non_value_no_seq(setting, filename): for i in xrange(200): yield [(i + 1) * (j + 1) for j in xrange(10)] @@ -77,28 +80,28 @@ def test_min_pool_size(setting, filename): yield random.randint(0, 100 - 1) -@provider(input_types=[index_slot(100, seq_type=SequenceType.SEQUENCE)], - can_over_batch_size=False, - calc_batch_size=lambda x: len(x[0])) +@provider( + input_types=[index_slot( + 100, seq_type=SequenceType.SEQUENCE)], + can_over_batch_size=False, + calc_batch_size=lambda x: len(x[0])) def test_can_over_batch_size(setting, filename): for _ in xrange(1 << 10): seq_len = random.randint(0, 99) yield [random.randint(0, 100 - 1) for _ in xrange(seq_len)] -@provider(input_types={'input1':index_slot(10), 'input2': index_slot(10)}) +@provider(input_types={'input1': index_slot(10), 'input2': index_slot(10)}) def test_input_order(setting, filename): for _ in xrange(1000): - yield { - 'input1': 0, - 'input2': 1 - } + yield {'input1': 0, 'input2': 1} -@provider(input_types=[index_slot(10)], - check=True, - check_fail_continue=True, - should_shuffle="123") # also test should shuffle +@provider( + input_types=[index_slot(10)], + check=True, + check_fail_continue=True, + should_shuffle="123") # also test should shuffle def test_check(settings, filename): yield_good_value = False @@ -108,4 +111,3 @@ def test_check(settings, filename): if i < 10: yield_good_value = True yield i - diff --git a/paddle/math/BaseMatrix.cu b/paddle/math/BaseMatrix.cu index d81b99e5441584b21fb023dcae65ccec7dd27996..54448bdb5a9bb4f665f28f973eada30a07fb5eee 100644 --- a/paddle/math/BaseMatrix.cu +++ b/paddle/math/BaseMatrix.cu @@ -1451,6 +1451,8 @@ int BaseMatrixT::applyRow(Agg agg, BaseMatrixT& b) { MatrixOffset offset(0, 0, 0, 0, 0, 0); int numRows = b.height_; int numCols = b.width_; + CHECK_EQ(height_, numRows); + CHECK_EQ(width_, 1UL); aggregate(agg, base::unary::identity(), base::binary::second(), b, numRows, numCols, offset, false_type(), true_type() /*aAsColVector*/); @@ -1463,18 +1465,69 @@ int BaseMatrixT::applyRow(Agg agg, Saver sv, BaseMatrixT& b) { MatrixOffset offset(0, 0, 0, 0, 0, 0); int numRows = b.height_; int numCols = b.width_; + CHECK_EQ(height_, numRows); + CHECK_EQ(width_, 1UL); aggregate(agg, base::unary::identity(), sv, b, numRows, numCols, offset, false_type(), true_type() /*aAsColVector*/); return 0; } +template<> +template +int BaseMatrixT::applyRow( + Agg agg, real scaleDest, real scaleAgg, BaseMatrixT& b) { + if (scaleDest != 0) { + applyRow(agg, base::binary::add2(scaleDest, scaleAgg), b); + } else { + applyRow(agg, base::binary::second(), b); + if (scaleAgg != 1) { + mulScalar(scaleAgg); + } + } + return 0; +} + +template<> +template +int BaseMatrixT::applyRow(Agg agg, Op op, Saver sv, + BaseMatrixT& b, BaseMatrixT& c) { + MatrixOffset offset(0, 0, 0, 0, 0, 0); + int numRows = b.height_; + int numCols = b.width_; + CHECK_EQ(height_, numRows); + CHECK_EQ(width_, 1UL); + CHECK_EQ(c.height_, numRows); + CHECK_EQ(c.width_, numCols); + aggregate(agg, op, sv, + b, c, numRows, numCols, offset, + false_type(), true_type() /*aAsColVector*/); + return 0; +} + +template<> +template +int BaseMatrixT::applyRow(Agg agg, Op op, real scaleDest, real scaleAgg, + BaseMatrixT& b, BaseMatrixT& c) { + if (scaleDest != 0) { + applyRow(agg, op, base::binary::add2(scaleDest, scaleAgg), b, c); + } else { + applyRow(agg, op, base::binary::second(), b, c); + if (scaleAgg != 1) { + mulScalar(scaleAgg); + } + } + return 0; +} + template<> template int BaseMatrixT::applyCol(Agg agg, BaseMatrixT& b) { MatrixOffset offset(0, 0, 0, 0, 0, 0); int numRows = b.height_; int numCols = b.width_; + CHECK_EQ(width_, numCols); + CHECK_EQ(height_, 1UL); aggregate(agg, base::unary::identity(), base::binary::second(), b, numRows, numCols, offset, true_type() /*aAsRowVector*/, false_type()); @@ -1487,6 +1540,8 @@ int BaseMatrixT::applyCol(Agg agg, Saver sv, BaseMatrixT& b) { MatrixOffset offset(0, 0, 0, 0, 0, 0); int numRows = b.height_; int numCols = b.width_; + CHECK_EQ(width_, numCols); + CHECK_EQ(height_, 1UL); aggregate(agg, base::unary::identity(), sv, b, numRows, numCols, offset, true_type() /*aAsRowVector*/, false_type()); @@ -1494,8 +1549,23 @@ int BaseMatrixT::applyCol(Agg agg, Saver sv, BaseMatrixT& b) { } template<> -void BaseMatrixT::sumRows(BaseMatrixT& b) { - applyRow(aggregate::sum(), b); +template +int BaseMatrixT::applyCol( + Agg agg, real scaleDest, real scaleAgg, BaseMatrixT& b) { + if (scaleDest != 0) { + applyCol(agg, base::binary::add2(scaleDest, scaleAgg), b); + } else { + applyCol(agg, base::binary::second(), b); + if (scaleAgg != 1) { + mulScalar(scaleAgg); + } + } + return 0; +} + +template<> +void BaseMatrixT::sumRows(BaseMatrixT& b, real scaleSum, real scaleDest) { + applyRow(aggregate::sum(), scaleDest, scaleSum, b); } template<> @@ -1524,18 +1594,22 @@ void BaseMatrixT::minCols(BaseMatrixT& b) { } template<> -void BaseMatrixT::sumCols(BaseMatrixT& b, real scale) { - applyCol(aggregate::sum(), base::binary::add2(1.0, scale), b); +void BaseMatrixT::sumCols(BaseMatrixT& b, real scaleSum, real scaleDest) { + applyCol(aggregate::sum(), scaleDest, scaleSum, b); } template<> -void BaseMatrixT::sumOfSquares(BaseMatrixT& b, BaseMatrixT& c) { - int numRows = b.height_; - int numCols = b.width_; - MatrixOffset offset(0, 0, 0, 0, 0, 0); - aggregate(aggregate::sum(), base::binary::squaredDiff(), base::binary::add(), - b, c, numRows, numCols, offset, false_type(), - true_type() /*aAsColVector*/); +void BaseMatrixT::sumOfSquaredDiffs( + BaseMatrixT& b, BaseMatrixT& c, real scaleSum, real scaleDest) { + applyRow(aggregate::sum(), base::binary::squaredDiff(), + scaleDest, scaleSum, b, c); +} + +template<> +void BaseMatrixT::sumOfProducts( + BaseMatrixT& b, BaseMatrixT& c, real scaleSum, real scaleDest) { + applyRow(aggregate::sum(), base::binary::mul(), + scaleDest, scaleSum, b, c); } template class BaseMatrixT; diff --git a/paddle/math/BaseMatrix.h b/paddle/math/BaseMatrix.h index 2dd2c2c7a9b985924d53cb3bf8840eb1e55eee3e..3a91fdc3c30c5332866a97c256b018eb0982260f 100644 --- a/paddle/math/BaseMatrix.h +++ b/paddle/math/BaseMatrix.h @@ -305,6 +305,23 @@ public: template int applyRow(Agg agg, BaseMatrixT& b); + /** + * a aggregate expression that apply each row of matrix b. + * + * @code + * for each row i & 0 <= j < b.width_, do: + * dst = agg(op(b[i*ldb + j], c[i*ldc + j]) + * this[i] = sv(this[i], dst) + * @endcode + */ + template + int applyRow(Agg agg, Op op, Saver sv, BaseMatrixT& b, BaseMatrixT& c); + + // Same as the above with the special handing of sv=add2(scaleDest, scaleAgg) + template + int applyRow(Agg agg, Op op, real scaleDest, real scaleAgg, + BaseMatrixT& b, BaseMatrixT& c); + /** * a aggregate expression that apply each row of matrix b. * @@ -317,6 +334,10 @@ public: template int applyRow(Agg agg, Saver sv, BaseMatrixT& b); + // Same as the above with the special handing of sv=add2(scaleDest, scaleAgg) + template + int applyRow(Agg agg, real scaleDest, real scaleAgg, BaseMatrixT& b); + /** * a aggregate expression that apply each column of matrix b. * @@ -340,6 +361,10 @@ public: template int applyCol(Agg agg, Saver sv, BaseMatrixT& b); + // Same as the above with the special handing of sv=add2(scaleDest, scaleAgg) + template + int applyCol(Agg agg, real scaleDest, real scaleAgg, BaseMatrixT& b); + bool useGpu() const { return useGpu_; } const T* rowBuf(size_t row) const { return data_ + width_ * row; } @@ -920,7 +945,9 @@ public: void addRowScale(size_t cCol, BaseMatrixT& b, BaseMatrixT& c); /// calculate the sum of each row of the matrix b. - void sumRows(BaseMatrixT& b); + /// this_i = scaleDest * this_i + scaleSum * \sum_j b_{ij} + void sumRows(BaseMatrixT& b, T scaleSum, T scaleDest); + /// calculate the maximum value of each row of the matrix b. void maxRows(BaseMatrixT& b); /// calculate the minimum value of each row of the matrix b. @@ -932,10 +959,18 @@ public: void maxCols(BaseMatrixT& b); /// calculate the minimum value of each column of the matrix b. void minCols(BaseMatrixT& b); - void sumCols(BaseMatrixT& b, T scale); - /// calculate the sum of each row of (b - c)^2. - void sumOfSquares(BaseMatrixT& b, BaseMatrixT& c); + /// calculate the sum of each column of the matrix b. + /// this_i = scaleDest * this_i + scaleSum * \sum_j b_{ji} + void sumCols(BaseMatrixT& b, T scaleSum, T scaleDest); + + /// this_i = scaleDest * this_i + scaleSum * \sum_j (b_{ij} - c_{ij})^2 + void sumOfSquaredDiffs(BaseMatrixT& b, BaseMatrixT& c, + T scaleSum, T scaleDest); + + /// this_i = scaleDest * this_i + scaleSum * \sum_j b_{ij} * c_{ij} + void sumOfProducts(BaseMatrixT& b, BaseMatrixT& c, + T scaleSum, T scaleDest); /** * @code diff --git a/paddle/math/CpuSparseMatrix.cpp b/paddle/math/CpuSparseMatrix.cpp index 842efdbe3d77ec3443374f62df5c520252aa7ce4..64ee124a5613a99ac3d7ff36897e4f2d0489ad51 100644 --- a/paddle/math/CpuSparseMatrix.cpp +++ b/paddle/math/CpuSparseMatrix.cpp @@ -409,9 +409,6 @@ void CpuSparseMatrix::setRow(size_t row, size_t colNum, if (format_ == SPARSE_CSR) { CHECK_LT(row, height_); CHECK(NULL != cols); - for (size_t i = row; i < height_; i++) { - CHECK_EQ(rows_[i + 1], rows_[i]); - } if (0 == row) { rows_[row] = 0; } diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index b322bd2bd719484b86b62bca5783d78bd8ca9a4c..29c07467c7bac9c382f02a5f6ffdcfd87c5b09a0 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -80,4 +80,3 @@ void vTanh(const int n, const T* a, T* r); } // namespace paddle #endif // MATHFUNCTIONS_H_ - diff --git a/paddle/math/MathUtils.cpp b/paddle/math/MathUtils.cpp index c1af8628d03c50185089b45f3a0502726da9137e..548f17936381c7e1c4d0c2c9661b197f3f06bd35 100644 --- a/paddle/math/MathUtils.cpp +++ b/paddle/math/MathUtils.cpp @@ -80,4 +80,17 @@ int outputSize(int imageSize, int filterSize, int padding, int stride, return outputSize; } +int imageSize(int outputSize, int filterSize, int padding, int stride, + bool caffeMode) { + int imageSize; + if (!caffeMode) { + imageSize = + (outputSize - 1) * stride + filterSize - 2 * padding - stride + 1; + } else { + imageSize = (outputSize - 1) * stride + filterSize - 2 * padding; + } + CHECK_GE(imageSize, 1); + return imageSize; +} + } // namespace paddle diff --git a/paddle/math/MathUtils.h b/paddle/math/MathUtils.h index 49d0c10a8f5e4dcdaf22dca77a3f113400b16646..91683dc3e9144df4664f46859ff5e2215dc34144 100644 --- a/paddle/math/MathUtils.h +++ b/paddle/math/MathUtils.h @@ -60,4 +60,11 @@ void sparseRand(int* major, int* minor, int nnz, int majorLen, int minorMax, int outputSize(int imageSize, int filterSize, int padding, int stride, bool caffeMode); +/** + * Calculate image size based on output size and caffeMode_. + * It is the reverse function of outputSize() + */ +int imageSize(int outputSize, int filterSize, int padding, int stride, + bool caffeMode); + } // namespace paddle diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 80bf74bd4c51a3776cba7fb683238ef3a147373e..706a598d0c33762b0578190ea4a0aa06247a88ef 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -13,20 +13,20 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "Matrix.h" +#include "MathFunctions.h" #include "SparseMatrix.h" #include "SparseRowMatrix.h" -#include "MathFunctions.h" -#include #include #include +#include -#include "paddle/utils/Logging.h" #include #include "hl_cnn.h" #include "hl_gpu.h" #include "hl_table_apply.h" #include "hl_top_k.h" +#include "paddle/utils/Logging.h" #include "paddle/utils/ThreadLocal.h" @@ -43,9 +43,9 @@ inline real _safelog(real a) { return a > 0.0f ? std::log(a) : -40.0f; } Matrix::Matrix(MemoryHandlePtr memHandle, size_t height, size_t width, bool trans, bool use_gpu) : BaseMatrix( - height, width, - memHandle ? (reinterpret_cast(memHandle->getBuf())) : nullptr, - trans, use_gpu) { + height, width, + memHandle ? (reinterpret_cast(memHandle->getBuf())) : nullptr, + trans, use_gpu) { elementCnt_ = width * height; memoryHandle_ = memHandle; } @@ -96,7 +96,7 @@ MatrixPtr Matrix::create(MemoryHandlePtr memHandle, size_t height, size_t width, if (auto gpuHandle = std::dynamic_pointer_cast(memHandle)) { return std::make_shared(gpuHandle, height, width, trans); } else if (auto cpuHandle = - std::dynamic_pointer_cast(memHandle)) { + std::dynamic_pointer_cast(memHandle)) { return std::make_shared(cpuHandle, height, width, trans); } else { LOG(FATAL) << "Wrong"; @@ -188,6 +188,15 @@ MatrixPtr Matrix::subMatrix(size_t startRow, size_t endRow, size_t startCol, trans_, useGpu_); } +void Matrix::setDiag(real value) { + CHECK(data_ != NULL); + CHECK_EQ(height_, width_); + + zeroMem(); + BaseMatrix diag(height_, 1, stride_ + 1, data_, false, useGpu_); + diag.assign(value); +} + GpuMatrix::GpuMatrix(size_t height, size_t width, bool trans) : Matrix(std::make_shared(height * width * sizeof(real)), height, width, trans, true) {} @@ -203,6 +212,7 @@ void GpuMatrix::resetOne() { CHECK(data_ != NULL); one(); } + void GpuMatrix::resize(size_t newHeight, size_t newWidth) { size_t newSize = newHeight * newWidth; if (NULL == memoryHandle_.get() || @@ -232,7 +242,7 @@ real GpuMatrix::getSum() { void GpuMatrix::accumulateColSum(Matrix& src) { CHECK_EQ(getWidth(), src.getWidth()); CHECK_EQ(getHeight(), (size_t)1); - sumCols(src, 1.0); + sumCols(src, 1.0, 1.0); } real GpuMatrix::getAbsSum() { @@ -377,17 +387,17 @@ void GpuMatrix::addSharedBias(Matrix& b, real scale) { void GpuMatrix::collectBias(Matrix& a, real scale) { CHECK_EQ(getHeight(), (size_t)1); CHECK_EQ(width_, a.getWidth()); - GpuSparseMatrix* sMatPtr = dynamic_cast(&a); + GpuSparseMatrix* sMatPtr = dynamic_cast(&a); if (!sMatPtr) { - sumCols(a, scale); + sumCols(a, /* scaleSum= */scale, /* scaleDest= */1); } else { real* data = getData(); hl_sparse_matrix_s A_d = sMatPtr->sMatrix_.get(); - hl_sparse_matrix_column_sum(data, A_d, sMatPtr->getHeight(), - width_, scale); + hl_sparse_matrix_column_sum(data, A_d, sMatPtr->getHeight(), width_, scale); } } + void GpuMatrix::collectSharedBias(Matrix& a, real scale) { CHECK_EQ(getHeight(), (size_t)1); CHECK_EQ(a.getWidth() % getWidth(), 0UL); @@ -443,8 +453,8 @@ void GpuMatrix::mul(const GpuMatrix& a, const GpuMatrix& b, real scaleAB, hl_trans_op_t transa = !a.isTransposed() ? HPPL_OP_N : HPPL_OP_T; hl_trans_op_t transb = !b.isTransposed() ? HPPL_OP_N : HPPL_OP_T; - hl_matrix_mul(A_d, transa, B_d, transb, C_d, dimM, dimN, dimK, - scaleAB, scaleT, lda, ldb, ldc); + hl_matrix_mul(A_d, transa, B_d, transb, C_d, dimM, dimN, dimK, scaleAB, + scaleT, lda, ldb, ldc); } void GpuMatrix::mul(const GpuSparseMatrix& a, const GpuMatrix& b, real scaleAB, @@ -465,8 +475,8 @@ void GpuMatrix::mul(const GpuSparseMatrix& a, const GpuMatrix& b, real scaleAB, hl_sparse_matrix_s A_d = a.sMatrix_.get(); real* B_d = b.data_; real* C_d = data_; - hl_matrix_csr_mul_dense(A_d, transA, B_d, HPPL_OP_N, C_d, height_, - width_, b.height_, scaleAB, scaleT); + hl_matrix_csr_mul_dense(A_d, transA, B_d, HPPL_OP_N, C_d, height_, width_, + b.height_, scaleAB, scaleT); } void GpuMatrix::mul(const GpuMatrix& a, const GpuSparseMatrix& b, real scaleAB, @@ -487,11 +497,11 @@ void GpuMatrix::mul(const GpuMatrix& a, const GpuSparseMatrix& b, real scaleAB, << "Matrix dimensions are not equal"; } if (b.format_ == SPARSE_CSC) { - hl_matrix_dense_mul_csc(A_d, HPPL_OP_N, B_d, transB, C_d, height_, - width_, a.width_, scaleAB, scaleT); + hl_matrix_dense_mul_csc(A_d, HPPL_OP_N, B_d, transB, C_d, height_, width_, + a.width_, scaleAB, scaleT); } else { - hl_matrix_dense_mul_csr(A_d, HPPL_OP_N, B_d, transB, C_d, height_, - width_, a.width_, scaleAB, scaleT); + hl_matrix_dense_mul_csr(A_d, HPPL_OP_N, B_d, transB, C_d, height_, width_, + a.width_, scaleAB, scaleT); } } @@ -553,8 +563,8 @@ void GpuMatrix::selectRows(Matrix& table, IVector& ids) { size_t tableSize = table.getHeight(); int* index = ids.getData(); - hl_matrix_select_rows(a, stride_, table.getData(), table.stride_, - index, numSamples, tableSize, dim); + hl_matrix_select_rows(a, stride_, table.getData(), table.stride_, index, + numSamples, tableSize, dim); #endif } @@ -571,15 +581,15 @@ void GpuMatrix::addToRows(Matrix& table, IVector& ids) { size_t tableSize = table.getHeight(); int* index = ids.getData(); - hl_matrix_add_to_rows(table.getData(), table.stride_, a, stride_, - index, numSamples, tableSize, dim); + hl_matrix_add_to_rows(table.getData(), table.stride_, a, stride_, index, + numSamples, tableSize, dim); #endif } void GpuMatrix::colMerge(Matrix& src) { CHECK(src.height_ == height_); if (!trans_ && !src.trans_) { - sumRows(src); + sumRows(src, /* scaleSum= */1, /* scaleDest= */0); } else { LOG(FATAL) << "Is not supported"; } @@ -589,7 +599,7 @@ void GpuMatrix::rowSum(Matrix& sum) { CHECK_EQ(sum.getHeight(), getHeight()); CHECK_EQ(sum.getWidth(), (size_t)1); - sum.sumRows(*this); + sum.sumRows(*this, /* scaleSum= */1, /* scaleDest= */0); } void GpuMatrix::rowMax(Matrix& max) { @@ -607,13 +617,8 @@ void GpuMatrix::rowMax(IVector& maxIds, Matrix& maxVal) { CHECK_EQ(maxIds.getSize(), numSamples * beam); CHECK_EQ(maxVal.getHeight(), numSamples); - hl_matrix_top_k(maxVal.getData(), - maxVal.getStride(), - maxIds.getData(), - this->getData(), - this->getStride(), - this->getWidth(), - beam, + hl_matrix_top_k(maxVal.getData(), maxVal.getStride(), maxIds.getData(), + this->getData(), this->getStride(), this->getWidth(), beam, numSamples); #endif } @@ -637,12 +642,12 @@ void GpuMatrix::maxoutForward(Matrix& a, IVector& id, size_t channels, size_t size = getWidth(); size_t batchSize = getHeight(); - const real* input = a.getData(); + const real* input = a.getData(); real* output = getData(); int* idForGpu = id.getData(); - hl_maxout_forward(input, output, idForGpu, batchSize, size, - size / channels, groups); + hl_maxout_forward(input, output, idForGpu, batchSize, size, size / channels, + groups); } void GpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, @@ -653,12 +658,12 @@ void GpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, size_t size = a.getWidth(); size_t batchSize = getHeight(); - real* input = getData(); + real* input = getData(); const real* output = a.getData(); const int* idForGpu = id.getData(); - hl_maxout_backward(input, output, idForGpu, batchSize, size, - size / channels, groups); + hl_maxout_backward(input, output, idForGpu, batchSize, size, size / channels, + groups); } /*calulate the error of classification */ @@ -674,8 +679,8 @@ void GpuMatrix::classificationError(MatrixPtr output, IVectorPtr label) { real* recResult_d = data_; int* label_d = label_ptr->getData(); - hl_matrix_classification_error(output_d, label_d, recResult_d, - height_, output_ptr->width_); + hl_matrix_classification_error(output_d, label_d, recResult_d, height_, + output_ptr->width_); } /* copy -log(output[i * width + label]) to this->data[i] */ @@ -744,8 +749,7 @@ void GpuMatrix::sequenceSoftmax(Matrix& output, const IVector& index) { real* outputData = output.getData(); auto starts = index.getData(); int numSequences = index.getSize() - 1; - hl_sequence_softmax_forward(inputData, outputData, - starts, numSequences); + hl_sequence_softmax_forward(inputData, outputData, starts, numSequences); } void GpuMatrix::softmaxDerivative(Matrix& output, Matrix& sftmaxSum) { @@ -759,8 +763,7 @@ void GpuMatrix::softmaxDerivative(Matrix& output, Matrix& sftmaxSum) { real* output_d = output.data_; real* sftmaxSum_d = sftmaxSum.data_; real* grad_d = data_; - hl_matrix_softmax_derivative(grad_d, output_d, sftmaxSum_d, height_, - width_); + hl_matrix_softmax_derivative(grad_d, output_d, sftmaxSum_d, height_, width_); } void GpuMatrix::softmaxBackward(Matrix& outputV) { @@ -787,7 +790,8 @@ void GpuMatrix::sumOfSquares(Matrix& output, Matrix& label) { LOG(FATAL) << "not supported: GpuSparseMatrix as label"; } - BaseMatrix::sumOfSquares(output, label); + BaseMatrix::sumOfSquaredDiffs(output, label, + /* scaleSum= */1, /* scaleDest= */1); } void GpuMatrix::sumOfSquaresBp(Matrix& outputV, Matrix& label) { @@ -811,7 +815,7 @@ void GpuMatrix::scaledTanh(Matrix& output, real p1, real p2) { } void GpuMatrix::cosSim(Matrix& output1, Matrix& output2, real scale) { CHECK(output1.useGpu_ == true && output2.useGpu_ == true) - << "Matrix type are not equal"; + << "Matrix type are not equal"; size_t numSamples = getHeight(); size_t dim = output1.getWidth(); CHECK_EQ(getWidth(), 1UL); @@ -820,15 +824,15 @@ void GpuMatrix::cosSim(Matrix& output1, Matrix& output2, real scale) { real* out = getData(); real* x = output1.getData(); real* y = output2.getData(); - hl_cossim(out, x, y, - dim, output1.getHeight(), output2.getHeight(), scale); + hl_cossim(out, x, y, dim, output1.getHeight(), output2.getHeight(), scale); } void GpuMatrix::cosSimDerivative(Matrix& output, Matrix& prevOut1, Matrix& prevOut2, Matrix& prevGrad1, Matrix& prevGrad2, real scale) { CHECK(output.useGpu_ == true && prevOut1.useGpu_ == true && prevOut2.useGpu_ == true && prevGrad1.useGpu_ == true && - prevGrad2.useGpu_ == true) << "Matrix type are not equal"; + prevGrad2.useGpu_ == true) + << "Matrix type are not equal"; CHECK_EQ(getWidth(), 1UL); CHECK_EQ(output.getWidth(), 1UL); @@ -848,9 +852,8 @@ void GpuMatrix::cosSimDerivative(Matrix& output, Matrix& prevOut1, real* prevOutY = prevOut2.getData(); real* prevGradX = prevGrad1.getData(); real* prevGradY = prevGrad2.getData(); - hl_cossim_derivative(grad, out, prevOutX, prevOutY, - prevGradX, prevGradY, dim, - prevOut1.getHeight(), prevOut2.getHeight(), scale); + hl_cossim_derivative(grad, out, prevOutX, prevOutY, prevGradX, prevGradY, dim, + prevOut1.getHeight(), prevOut2.getHeight(), scale); } void GpuMatrix::randomizeUniform() { @@ -901,8 +904,8 @@ void GpuMatrix::check(std::ostream& os, Matrix& refMat, bool printDiff) { void GpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, int channels, int blockH, int blockW, int strideH, - int strideW, int paddingH, int paddingW, - int outputH, int outputW) { + int strideW, int paddingH, int paddingW, int outputH, + int outputW) { CHECK(feature.useGpu_ == true) << "Matrix type are not equal"; CHECK_EQ(size_t(feaImgHeight * feaImgWidth * channels), @@ -912,17 +915,16 @@ void GpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, size_t elemCnt = outputH * outputW * blockH * blockW * channels; CHECK_EQ(elemCnt, height_ * width_) << "Matrix dimensions are not equal"; - hl_expand_feature2col(feature.getData(), channels, feaImgHeight, - feaImgWidth, blockH, blockW, strideH, strideW, - paddingH, paddingW, outputH, outputW, - getData()); + hl_expand_feature2col(feature.getData(), channels, feaImgHeight, feaImgWidth, + blockH, blockW, strideH, strideW, paddingH, paddingW, + outputH, outputW, getData()); } void GpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, int thisImgWidth, int channels, int blockH, int blockW, int strideH, int strideW, int paddingH, - int paddingW, int outputH, int outputW, - real alpha, real beta) { + int paddingW, int outputH, int outputW, real alpha, + real beta) { CHECK(expandFeat.useGpu_ == true) << "Matrix type are not equal"; CHECK_EQ(size_t(thisImgHeight * thisImgWidth * channels), getHeight() * getWidth()) @@ -931,18 +933,17 @@ void GpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, size_t elemCnt = outputH * outputW * blockW * blockH * channels; CHECK(elemCnt == expandFeat.getHeight() * expandFeat.getWidth()) << "Matrix dimensions are not equal"; - hl_shrink_col2feature( - expandFeat.getData(), channels, thisImgHeight, thisImgWidth, blockH, - blockW, strideH, strideW, paddingH, paddingW, outputH, outputW, - getData(), alpha, beta); + hl_shrink_col2feature(expandFeat.getData(), channels, thisImgHeight, + thisImgWidth, blockH, blockW, strideH, strideW, + paddingH, paddingW, outputH, outputW, getData(), alpha, + beta); } void GpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - size_t paddingH, size_t paddingW) { + size_t imgSizeW, size_t channels, size_t sizeX, + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { CHECK(inputMat.useGpu_ == true) << "Matrix type are not equal"; real* inputData = inputMat.getData(); @@ -953,16 +954,15 @@ void GpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(height_ == inputMat.getHeight()); CHECK(width_ == outputH * outputW * channels); - hl_maxpool_forward(frameNum, inputData, channels, height, width, - outputH, outputW, sizeX, sizeY, strideH, strideW, - paddingH, paddingW, data_); + hl_maxpool_forward(frameNum, inputData, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, data_, getStride()); } void GpuMatrix::maxPoolBackward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, Matrix& outGrad, Matrix& outV, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, + size_t sizeX, size_t sizeY, size_t strideH, + size_t strideW, size_t outputH, size_t outputW, real scaleTargets, real scaleOutput, size_t paddingH, size_t paddingW) { CHECK(inputMat.useGpu_ == true && outGrad.useGpu_ == true && @@ -982,19 +982,17 @@ void GpuMatrix::maxPoolBackward(Matrix& inputMat, size_t imgSizeH, CHECK(outGrad.getHeight() == outV.getHeight() && outGrad.getWidth() == outV.getWidth()); - - hl_maxpool_backward(frameNum, inputData, outData, outDiff, channels, - height, width, outputH, outputW, sizeX, sizeY, - strideH, strideW, paddingH, paddingW, - scaleTargets, scaleOutput, data_); + hl_maxpool_backward(frameNum, inputData, outData, outDiff, channels, height, + width, outputH, outputW, sizeX, sizeY, strideH, strideW, + paddingH, paddingW, scaleTargets, scaleOutput, data_, + outGrad.getStride()); } void GpuMatrix::avgPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - size_t paddingH, size_t paddingW) { + size_t imgSizeW, size_t channels, size_t sizeX, + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { CHECK(inputMat.useGpu_ == true) << "Matrix type are not equal"; real* inputData = inputMat.getData(); @@ -1005,18 +1003,17 @@ void GpuMatrix::avgPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(height_ == inputMat.getHeight()); CHECK(width_ == outputH * outputW * channels); - hl_avgpool_forward(frameNum, inputData, channels, height, width, - outputH, outputW, sizeX, sizeY, - strideH, strideW, - paddingH, paddingW, data_); + hl_avgpool_forward(frameNum, inputData, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, data_, getStride()); } void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, size_t imgSizeW, size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput, - size_t paddingH, size_t paddingW) { + size_t strideH, size_t strideW, size_t outputH, + size_t outputW, real scaleTargets, + real scaleOutput, size_t paddingH, + size_t paddingW) { CHECK(outGrad.useGpu_ == true) << "Matrix type are not equal"; real* outDiff = outGrad.getData(); @@ -1028,11 +1025,10 @@ void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, CHECK(height_ == outGrad.getHeight()); CHECK(outGrad.getWidth() == outputH * outputW * channels); - hl_avgpool_backward(frameNum, outDiff, channels, height, width, - outputH, outputW, sizeX, sizeY, - strideH, strideW, paddingH, paddingW, - scaleTargets, scaleOutput, - data_); + hl_avgpool_backward(frameNum, outDiff, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, scaleTargets, scaleOutput, data_, + outGrad.getStride()); } void GpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, @@ -1047,8 +1043,8 @@ void GpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, CHECK(denoms.getHeight() == input.getHeight() && denoms.getWidth() == input.getWidth() && input.getHeight() == height_ && input.getWidth() == width_); - hl_CMRNorm_forward(num, input.getData(), denoms.getData(), data_, - channels, height, width, sizeX, scale, -pow); + hl_CMRNorm_forward(num, input.getData(), denoms.getData(), data_, channels, + height, width, sizeX, scale, -pow); } void GpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, @@ -1068,13 +1064,11 @@ void GpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, denoms.getWidth() == localGrad.getWidth()); hl_CMRNorm_backward(num, preOutV.getData(), denoms.getData(), - localOutV.getData(), localGrad.getData(), data_, - channels, height, width, sizeX, -pow, - 2.0f * pow * scale); + localOutV.getData(), localGrad.getData(), data_, channels, + height, width, sizeX, -pow, 2.0f * pow * scale); } -void GpuMatrix::maxSequenceForward(Matrix& input, - const IVector& sequence, +void GpuMatrix::maxSequenceForward(Matrix& input, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&input)); CHECK(dynamic_cast(&sequence)); @@ -1091,12 +1085,11 @@ void GpuMatrix::maxSequenceForward(Matrix& input, CHECK_EQ(numSequences, sequence.getSize() - 1); CHECK_EQ(numSequences * dim, index.getSize()); - hl_max_sequence_forward(inputData, starts, outData, maxIndex, - numSequences, dim); + hl_max_sequence_forward(inputData, starts, outData, maxIndex, numSequences, + dim); } -void GpuMatrix::maxSequenceBackward(Matrix& outputGrad, - const IVector& sequence, +void GpuMatrix::maxSequenceBackward(Matrix& outputGrad, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&outputGrad)); CHECK(dynamic_cast(&sequence)); @@ -1153,9 +1146,8 @@ void GpuMatrix::contextProjectionBackwardData(MatrixPtr inputGrad, real* inGrad = inputGrad->getData(); const int* starts = sequence.getData(); - hl_context_projection_backward_data(outGrad, starts, inGrad, - numSequences, inputDim, - contextLength, contextStart); + hl_context_projection_backward_data(outGrad, starts, inGrad, numSequences, + inputDim, contextLength, contextStart); } void GpuMatrix::contextProjectionBackwardWeight(MatrixPtr weightGrad, @@ -1175,9 +1167,9 @@ void GpuMatrix::contextProjectionBackwardWeight(MatrixPtr weightGrad, real* wtGrad = weightGrad->getData(); const int* starts = sequence.getData(); - hl_context_projection_backward_weight( - outGrad, starts, wtGrad, numSequences, weightDim, totalPad, contextLength, - contextStart, beginPad); + hl_context_projection_backward_weight(outGrad, starts, wtGrad, numSequences, + weightDim, totalPad, contextLength, + contextStart, beginPad); } void GpuMatrix::paramReluForward(Matrix& data, Matrix& W) { @@ -1189,8 +1181,7 @@ void GpuMatrix::paramReluForward(Matrix& data, Matrix& W) { size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (W.getHeight() * W.getWidth()); real* output = getData(); - hl_param_relu_forward(output, input, w, numElements, numSamples, - partial_sum); + hl_param_relu_forward(output, input, w, numElements, numSamples, partial_sum); } void GpuMatrix::paramReluBackwardW(Matrix& oGrad, Matrix& data) { @@ -1202,8 +1193,8 @@ void GpuMatrix::paramReluBackwardW(Matrix& oGrad, Matrix& data) { size_t numElements = data.getWidth(); size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (this->getHeight() * this->getWidth()); - hl_param_relu_backward_w(wgrad, ograd, input, - numElements, numSamples, partial_sum); + hl_param_relu_backward_w(wgrad, ograd, input, numElements, numSamples, + partial_sum); } void GpuMatrix::paramReluBackwardDiff(Matrix& oGrad, Matrix& data, Matrix& W) { @@ -1214,8 +1205,8 @@ void GpuMatrix::paramReluBackwardDiff(Matrix& oGrad, Matrix& data, Matrix& W) { size_t numElements = data.getWidth(); size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (W.getHeight() * W.getWidth()); - hl_param_relu_backward_diff(ograd, input, w, diff, - numElements, numSamples, partial_sum); + hl_param_relu_backward_diff(ograd, input, w, diff, numElements, numSamples, + partial_sum); } void GpuMatrix::addColumnVector(const Matrix& b) { @@ -1278,6 +1269,42 @@ void GpuMatrix::bilinearBackward(const Matrix& out, } } +void GpuMatrix::multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label) { + GpuMatrix* outputPtr = dynamic_cast(&output); + auto labelPtr = dynamic_cast(&label); + + CHECK(outputPtr && labelPtr) << "Invalid argument pointer"; + CHECK(labelPtr->format_ == SPARSE_CSR) << "Matrix format not supported"; + CHECK(height_ == outputPtr->height_ && width_ == 1 + && outputPtr->width_ == labelPtr->getWidth() + && outputPtr->height_ == labelPtr->getHeight()) + << "Matrix dimensions are not equal"; + + real* output_d = outputPtr->data_; + real* entropy_d = data_; + hl_sparse_matrix_s mat_d = labelPtr->sMatrix_.get(); + hl_matrix_multi_binary_cross_entropy( + output_d, entropy_d, mat_d, height_, outputPtr->width_); +} + +void GpuMatrix::multiBinaryLabelCrossEntropyBp(Matrix &output, Matrix &label) { + GpuMatrix* outputPtr = dynamic_cast(&output); + auto labelPtr = dynamic_cast(&label); + + CHECK(outputPtr && labelPtr) << "Invalid argument pointer"; + CHECK(labelPtr->format_ == SPARSE_CSR) << "Matrix format not supported"; + CHECK(height_ == outputPtr->height_ && width_ == outputPtr->width_ + && outputPtr->width_ == labelPtr->getWidth() + && outputPtr->height_ == labelPtr->getHeight()) + << "Matrix dimensions are not equal"; + + real* output_d = outputPtr->data_; + real* grad_d = data_; + hl_sparse_matrix_s mat_d = labelPtr->sMatrix_.get(); + hl_matrix_multi_binary_cross_entropy_bp( + output_d, grad_d, mat_d, height_, width_); +} + /** * CpuMatrix */ @@ -1475,7 +1502,7 @@ void CpuMatrix::accumulateColSum(Matrix& src) { CHECK_EQ(getWidth(), src.getWidth()); CHECK_EQ(getHeight(), (size_t)1); - sumCols(src, 1.0); + sumCols(src, /* scaleSum= */1, /* scaleDest= */1); } real CpuMatrix::getAbsSum() { @@ -1561,8 +1588,8 @@ void CpuMatrix::inverse(MatrixPtr matInv, bool memAlloc) { void CpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, int channels, int blockH, int blockW, int strideH, - int strideW, int paddingH, int paddingW, - int outputH, int outputW) { + int strideW, int paddingH, int paddingW, int outputH, + int outputW) { CHECK(feature.useGpu_ == false) << "Matrix type are not equal"; CHECK_EQ(size_t(feaImgHeight * feaImgWidth * channels), @@ -1602,8 +1629,8 @@ void CpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, void CpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, int thisImgWidth, int channels, int blockH, int blockW, int strideH, int strideW, int paddingH, - int paddingW, int outputH, int outputW, - real alpha, real beta) { + int paddingW, int outputH, int outputW, real alpha, + real beta) { CHECK(expandFeat.useGpu_ == false) << "Matrix type are not equal"; CHECK_EQ(size_t(thisImgHeight * thisImgWidth * channels), getHeight() * getWidth()) @@ -1640,11 +1667,10 @@ void CpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, } void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - size_t paddingH, size_t paddingW) { + size_t imgSizeW, size_t channels, size_t sizeX, + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { real* inputData = inputMat.getData(); real* outData = data_; size_t num = inputMat.getHeight(); @@ -1652,15 +1678,21 @@ void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t inHeight = imgSizeH; CHECK(inHeight * inWidth == inputMat.getWidth() / channels); CHECK_EQ(num, this->getHeight()); - CHECK_EQ(channels*outputH*outputW, this->getWidth()); + CHECK_EQ(channels * outputH * outputW, this->getWidth()); + size_t outStride = getStride(); /* initialize the data_ */ - for (size_t i = 0; i < height_ * width_; i++) { - outData[i] = -(real)FLT_MAX; + for (size_t i = 0; i < height_; i++) { + for (size_t j = 0; j < width_; j++) { + outData[i * outStride + j] = -(real)FLT_MAX; + } } /* pool max one by one */ - for (size_t n = 0; n < num; ++n) { // frame by frame + for (size_t n = 0; n < num; ++n) { // frame by frame + if (!isContiguous()) { + outData = data_ + n * outStride; + } for (size_t c = 0; c < channels; ++c) { // channel by channel for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { @@ -1702,7 +1734,16 @@ void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, real* inData = image.getData(); real* otData = outV.getData(); real* otGrad = outGrad.getData(); + + size_t outStride = outV.getStride(); + real* origOutData = otData; + real* origOutGrad = otGrad; + for (size_t n = 0; n < num; ++n) { + if (!outV.isContiguous()) { + otData = origOutData + n * outStride; + otGrad = origOutGrad + n * outStride; + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { @@ -1733,9 +1774,9 @@ void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, size_t channels, size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - size_t paddingH, size_t paddingW) { + size_t strideH, size_t strideW, size_t outputH, + size_t outputW, size_t paddingH, + size_t paddingW) { // The main loop size_t num = input.getHeight(); size_t inHeight = imgSizeH; @@ -1746,6 +1787,9 @@ void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, real* inData = input.getData(); for (size_t n = 0; n < num; ++n) { + if (!isContiguous()) { + tgtData = data_ + n * getStride(); + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { @@ -1777,9 +1821,8 @@ void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, } void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, + size_t sizeX, size_t sizeY, size_t strideH, + size_t strideW, size_t outputH, size_t outputW, real scaleTargets, real scaleOutput, size_t paddingH, size_t paddingW) { size_t num = input.getHeight(); @@ -1789,6 +1832,9 @@ void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, real* outData = getData(); for (size_t n = 0; n < num; ++n) { + if (!input.isContiguous()) { + inData = input.getData() + n * input.getStride(); + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { @@ -1891,8 +1937,7 @@ void CpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, * Output: output size is the number of input sequences (NOT input instances). * output[i] is set to max_{for each instance in this sequence}{input[i]} */ -void CpuMatrix::maxSequenceForward(Matrix& input, - const IVector& sequence, +void CpuMatrix::maxSequenceForward(Matrix& input, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&input)); CHECK(dynamic_cast(&sequence)); @@ -1933,8 +1978,7 @@ void CpuMatrix::maxSequenceForward(Matrix& input, } } -void CpuMatrix::maxSequenceBackward(Matrix& outputGrad, - const IVector& sequence, +void CpuMatrix::maxSequenceBackward(Matrix& outputGrad, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&outputGrad)); CHECK(dynamic_cast(&sequence)); @@ -2145,7 +2189,7 @@ void CpuMatrix::collectBias(Matrix& a, real scale) { CHECK_EQ(width_, a.getWidth()); CpuSparseMatrix* aptr = dynamic_cast(&a); if (!aptr) { - sumCols(a, scale); + sumCols(a, /* scaleSum= */scale, /* scaleDest= */1); } else { size_t nnz = aptr->getElementCnt(); int* cols = aptr->getCols(); @@ -2184,7 +2228,7 @@ void CpuMatrix::sequenceAvgForward(Matrix& a, real* dst = getData(); real* src = a.getData(); const int* starts = startsPos.getData(); - MatrixPtr outMtx = Matrix::create(1, 1, false, false); + MatrixPtr outMtx = Matrix::create(nullptr, 1, width, false, false); MatrixPtr dataMtx = Matrix::create(nullptr, 1, width, false, false); for (size_t i = 0; i < height; i++) { int sequenceLength = starts[i + 1] - starts[i]; @@ -2196,13 +2240,15 @@ void CpuMatrix::sequenceAvgForward(Matrix& a, dataMtx->setData(src + starts[i] * width, sequenceLength, width); if (mode == 0) { // plain average - outMtx->sumCols(*dataMtx, (real)1 / (real)sequenceLength); + outMtx->sumCols(*dataMtx, (real)1 / (real)sequenceLength, + /* scaleDest= */1); } else if (mode == 1) { // sum instead of average - outMtx->sumCols(*dataMtx, (real)1); + outMtx->sumCols(*dataMtx, /* scaleSum= */1, /* scaleDest= */1); } else if (mode == 2) { // divide by square root of sequenceLength - outMtx->sumCols(*dataMtx, (real)1 / std::sqrt(sequenceLength)); + outMtx->sumCols(*dataMtx, (real)1 / std::sqrt(sequenceLength), + /* scaleDest= */1); } else { LOG(FATAL) << "should not reach here"; } @@ -2766,7 +2812,7 @@ void SharedCpuMatrix::mul(CpuSparseMatrix* a, CpuMatrix* b, real scaleAB, blockSeq.push_back(k); } std::shuffle(blockSeq.begin(), blockSeq.end(), - ThreadLocalRandomEngine::get()); + ThreadLocalRandomEngine::get()); } std::vector& localBufRows = *localBufRows_; int* cols = a->getCols(); @@ -2889,7 +2935,7 @@ void CpuMatrix::rowSum(Matrix& sum) { CHECK_EQ(sum.getHeight(), getHeight()); CHECK_EQ(sum.getWidth(), (size_t)1); - sum.sumRows(*this); + sum.sumRows(*this, /* scaleSum= */1, /* scaleDest= */0); } void CpuMatrix::rowMaxId(IVector& maxIds) { @@ -2997,7 +3043,7 @@ void CpuMatrix::maxoutForward(Matrix& a, IVector& id, size_t channels, size_t size = getWidth(); size_t batchSize = getHeight(); size_t featLen = size / channels; - const real* input = a.getData(); + const real* input = a.getData(); int* idForCpu = id.getData(); MatrixPtr maxInMat, maxOutMat; @@ -3031,8 +3077,8 @@ void CpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, size_t batchSize = getHeight(); size_t featLen = size / channels; size_t newFeatLen = groups * featLen; - real* inputG = getData(); - const real* outG = a.getData(); + real* inputG = getData(); + const real* outG = a.getData(); int* idForCpu = id.getData(); for (size_t batch_idx = 0; batch_idx < batchSize; ++batch_idx) { @@ -3256,9 +3302,9 @@ void CpuMatrix::sequenceSoftmax(Matrix& output, const IVector& index) { CHECK(isContiguous()); MatrixPtr inTmp = Matrix::create(nullptr, /* height= */ 1, 1, - /* trans= */ false, false); + /* trans= */ false, false); MatrixPtr outTmp = Matrix::create(nullptr, /* height= */ 1, 1, - /* trans= */ false, false); + /* trans= */ false, false); size_t numSequences = index.getSize() - 1; auto starts = index.getData(); for (size_t i = 0; i < numSequences; ++i) { @@ -3442,7 +3488,8 @@ void CpuMatrix::sumOfSquares(Matrix& output, Matrix& label) { } } - BaseMatrix::sumOfSquares(output, label); + BaseMatrix::sumOfSquaredDiffs(output, label, + /* scaleSum= */1, /* scaleDest= */1); } /* calculate the error of outputV according to label */ diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 07a2aebf556ef76e0f2a4432d8e709b4e4803842..6c3c4804d2fc67a3378c61e8b8ff1e3c0087dd83 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -195,6 +195,8 @@ public: virtual void resetOne() { LOG(FATAL) << "Not implemented"; } + void setDiag(real value); + virtual void copyFrom(const Matrix& src) { LOG(FATAL) << "Not implemented"; } virtual void trimFrom(const CpuSparseMatrix& src) { @@ -330,6 +332,7 @@ public: virtual MatrixPtr getInverse() { LOG(FATAL) << "Not implemented"; + return nullptr; } /** @@ -1036,6 +1039,7 @@ public: void zeroMem(); void resetOne(); + void setDiag(real value); void resize(size_t newHeight, size_t newWidth); void resize(size_t newHeight, size_t newWidth, @@ -1299,6 +1303,10 @@ public: const size_t numChannels, const real ratioH, const real ratioW); + + void multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label); + + void multiBinaryLabelCrossEntropyBp(Matrix& output, Matrix& label); }; class CpuMatrix : public Matrix { @@ -1318,6 +1326,8 @@ public: void zeroMem(); void resetOne(); + void setDiag(real value); + void resize(size_t newHeight, size_t newWidth); void resize(size_t newHeight, size_t newWidth, size_t newNnz, /* used to allocate space */ diff --git a/paddle/math/Vector.cpp b/paddle/math/Vector.cpp index 7553ea25e09d2f52f1f8b9205f954510b77cbfa9..23c9cacceaea2a2e265108b2467c1d21a2fe312f 100644 --- a/paddle/math/Vector.cpp +++ b/paddle/math/Vector.cpp @@ -21,6 +21,7 @@ limitations under the License. */ #include "paddle/utils/ThreadLocal.h" #include "paddle/utils/Thread.h" #include "paddle/utils/Flags.h" +#include "Matrix.h" #include "hl_gpu.h" #include "hl_table_apply.h" @@ -73,6 +74,31 @@ std::shared_ptr> VectorT::create(size_t size, } } +template <> +MatrixPtr VectorT::toOneHotSparseMatrix(size_t idRange, bool useGpu) { + LOG(FATAL) << "Wrong for real vector"; + return nullptr; +} + +template <> +MatrixPtr VectorT::toOneHotSparseMatrix(size_t idRange, bool useGpu) { + int height = getSize(); + int width = idRange; + MatrixPtr mat = Matrix::createSparseMatrix( + height, idRange, height, NO_VALUE, SPARSE_CSR, false, useGpu); + + CpuIVector cpuIds(height); + cpuIds.copyFrom(*this); + int *idData = cpuIds.getData(); + + for (int i = 0; i < height; i ++) { + const unsigned int id = idData[i]; + CHECK_LT(id, width); + mat->setRow(i, 1, &id, nullptr); + } + return mat; +} + template GpuVectorT::GpuVectorT(size_t size) : VectorT(size, std::make_shared(sizeof(T) * size), diff --git a/paddle/math/Vector.h b/paddle/math/Vector.h index ee0a83bf038f04ee9f7b3561639aa90da68a6e29..faf8186b6d10d7cbc14376ff3b6543d1303b2ab1 100644 --- a/paddle/math/Vector.h +++ b/paddle/math/Vector.h @@ -37,6 +37,8 @@ class BaseVector; class SyncThreadPool; +class Matrix; + template class BaseVector : public BaseMatrixT { public: @@ -155,6 +157,12 @@ public: subVecFrom(src, interval.first, interval.second - interval.first); } + /** + * convert the vector to a sparse one_hot matrix of width idRange + * only applies to IVector + */ + std::shared_ptr toOneHotSparseMatrix(size_t idRange, bool useGpu); + /** * This function will crash if the size of src and dest is different. */ diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index 017fddc799591de1c5acda6e5738591cc64aad7d..9c03695ba5055c4bdb3e7c578d3e352fbd6fae6f 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -714,20 +714,23 @@ void testMatrixInverse(int height) { MatrixPtr cpuI = std::make_shared(height, height); MatrixPtr gpuI = std::make_shared(height, height); + /* Make matrix well conditioned: cpu * cpuT + Identity */ cpu->randomizeUniform(); + MatrixPtr cpuT = cpu->getTranspose(); + MatrixPtr outputCheck = std::make_shared(height, height); + outputCheck->mul(cpu, cpuT); + cpu->setDiag(1.0); + cpu->add(*outputCheck); + gpu->copyFrom(*cpu); cpu->inverse(cpuI, false); gpu->inverse(gpuI, false); - MatrixPtr outputCheck = std::make_shared(height, height); outputCheck->copyFrom(*gpuI); MatrixCheckErr(*cpuI, *outputCheck); outputCheck->mul(cpu, cpuI); - cpu->zeroMem(); - for (int i = 0; i < height; i++) { - cpu->getRowBuf(i)[i] = 1.0; - } + cpu->setDiag(1.0); MatrixCheckErr(*cpu, *outputCheck); } @@ -2205,7 +2208,6 @@ void testCollectSharedBias(int numSamples, int dim, int channel) { MatrixCheckErr(*cpuBias, *check); } - TEST(Matrix, sharedBias) { for (auto numSamples : {1, 100, 520}) { for (auto dim : {100 * 16, 100 * 32}) { @@ -2219,6 +2221,60 @@ TEST(Matrix, sharedBias) { } } +void testMultiBinaryLabelCrossEntropy(int numSamples, int dim) { + MatrixPtr output = std::make_shared(numSamples, dim); + MatrixPtr cpuOutput = std::make_shared(numSamples, dim); + MatrixPtr gpuOutput = std::make_shared(numSamples, dim); + + MatrixPtr cpuEntropy = std::make_shared(numSamples, 1); + MatrixPtr gpuEntropy = std::make_shared(numSamples, 1); + + MatrixPtr cpuGrad = std::make_shared(numSamples, dim); + MatrixPtr gpuGrad = std::make_shared(numSamples, dim); + + MatrixPtr cpuLabel = std::make_shared + (numSamples, dim, numSamples, NO_VALUE, SPARSE_CSR, false); + MatrixPtr gpuLabel = std::make_shared + (numSamples, dim, numSamples, NO_VALUE, SPARSE_CSR, false); + for (int i = 0; i < numSamples; i ++) { + const unsigned int id = rand() % dim; // NOLINT + cpuLabel->setRow(i, 1, &id, nullptr); + gpuLabel->setRow(i, 1, &id, nullptr); + } + + output->randomizeUniform(); + cpuOutput->zeroMem(); + output->softmax(*cpuOutput); + gpuOutput->copyFrom(*cpuOutput); + + cpuEntropy->zeroMem(); + gpuEntropy->zeroMem(); + cpuEntropy->multiBinaryLabelCrossEntropy(*cpuOutput, *cpuLabel); + gpuEntropy->multiBinaryLabelCrossEntropy(*gpuOutput, *gpuLabel); + + MatrixPtr check1 = std::make_shared(numSamples, 1); + check1->copyFrom(*gpuEntropy); + MatrixCheckErr(*cpuEntropy, *check1); + + cpuGrad->zeroMem(); + gpuGrad->zeroMem(); + cpuGrad->multiBinaryLabelCrossEntropyBp(*cpuOutput, *cpuLabel); + gpuGrad->multiBinaryLabelCrossEntropyBp(*gpuOutput, *gpuLabel); + + MatrixPtr check2 = std::make_shared(numSamples, dim); + check2->copyFrom(*gpuGrad); + MatrixCheckErr(*cpuGrad, *check2); +} + +TEST(Matrix, multiBinaryCrossEntropy) { + for (auto numSamples : {100, 1000, 10000}) { + for (auto dim : {100, 1000, 10000}) { + VLOG(3) << " numSamples=" << numSamples << " dim=" << dim; + testMultiBinaryLabelCrossEntropy(numSamples, dim); + } + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/parameter/CMakeLists.txt b/paddle/parameter/CMakeLists.txt index d6f67604c03485dfeb3c907705b117ac550e9b6f..a35e46997fb04e9378e106bf428a629b286c2e8c 100644 --- a/paddle/parameter/CMakeLists.txt +++ b/paddle/parameter/CMakeLists.txt @@ -10,4 +10,4 @@ add_style_check_target(paddle_parameter ${PARAMETERS_HEADERS}) add_dependencies(paddle_parameter gen_proto_cpp) if(WITH_TESTING) add_subdirectory(tests) -endif() \ No newline at end of file +endif() diff --git a/paddle/parameter/tests/CMakeLists.txt b/paddle/parameter/tests/CMakeLists.txt index 177fb2fdfc045e1c68cc56e9f7654cbda5f46e25..cab264db8e5000e8eb61830ec07e9f590c103119 100644 --- a/paddle/parameter/tests/CMakeLists.txt +++ b/paddle/parameter/tests/CMakeLists.txt @@ -1 +1 @@ -add_simple_unittest(test_common) \ No newline at end of file +add_simple_unittest(test_common) diff --git a/paddle/py_paddle/__init__.py b/paddle/py_paddle/__init__.py index f372068942ea36a05c1433b482731bf112bfa51e..f8399f9c63d81f5a52bf2b277789c26d809f0153 100644 --- a/paddle/py_paddle/__init__.py +++ b/paddle/py_paddle/__init__.py @@ -15,9 +15,10 @@ from util import DataProviderWrapperConverter from dataprovider_converter import DataProviderConverter -__all__ = ['paddle', - 'DataProviderConverter', - 'DataProviderWrapperConverter', # for deprecated usage. - 'loadParameterFile'] +__all__ = [ + 'paddle', + 'DataProviderConverter', + 'DataProviderWrapperConverter', # for deprecated usage. + 'loadParameterFile' +] util.monkeypatches() - diff --git a/paddle/py_paddle/dataprovider_converter.py b/paddle/py_paddle/dataprovider_converter.py index dd2e146d112c055a68c8279417ce07d06fa10a7e..d64c7b20cb65a4b8dfebfc516cfc2c3fdc247114 100644 --- a/paddle/py_paddle/dataprovider_converter.py +++ b/paddle/py_paddle/dataprovider_converter.py @@ -45,10 +45,8 @@ class DenseScanner(IScanner): def finish_scan(self, argument): assert isinstance(argument, swig_paddle.Arguments) assert isinstance(self.input_type, dp2.InputType) - m = swig_paddle.Matrix.createDense(self.__mat__, - self.__height__, - self.input_type.dim, - False) + m = swig_paddle.Matrix.createDense(self.__mat__, self.__height__, + self.input_type.dim, False) argument.setSlotValue(self.pos, m) @@ -141,8 +139,10 @@ class DataProviderConverter(object): assert isinstance(argument, swig_paddle.Arguments) argument.resize(len(self.input_types)) - scanners = [DataProviderConverter.create_scanner(i, each_type) - for i, each_type in enumerate(self.input_types)] + scanners = [ + DataProviderConverter.create_scanner(i, each_type) + for i, each_type in enumerate(self.input_types) + ] for each_sample in dat: for each_step, scanner in zip(each_sample, scanners): @@ -171,11 +171,14 @@ class DataProviderConverter(object): assert retv is not None if each.seq_type == dp2.SequenceType.SUB_SEQUENCE: - retv = SequenceScanner(each, i, retv, lambda a, p, seq: - a.setSlotSubSequenceStartPositions(p, seq)) - - if each.seq_type in [dp2.SequenceType.SUB_SEQUENCE, - dp2.SequenceType.SEQUENCE]: - retv = SequenceScanner(each, i, retv, lambda a, p, seq: - a.setSlotSequenceStartPositions(p, seq)) + retv = SequenceScanner( + each, i, retv, + lambda a, p, seq: a.setSlotSubSequenceStartPositions(p, seq)) + + if each.seq_type in [ + dp2.SequenceType.SUB_SEQUENCE, dp2.SequenceType.SEQUENCE + ]: + retv = SequenceScanner( + each, i, retv, + lambda a, p, seq: a.setSlotSequenceStartPositions(p, seq)) return retv diff --git a/paddle/py_paddle/util.py b/paddle/py_paddle/util.py index 53f67a861e7d972648cfd22f451c6e56fa5aa149..e1f310580f95cfb210ba89589bab668433818b23 100644 --- a/paddle/py_paddle/util.py +++ b/paddle/py_paddle/util.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Some Useful method for py_paddle. """ @@ -79,20 +78,19 @@ class __ParameterCallbackWrapper__(swig_paddle.UpdateCallback): else: return __ParameterCallbackWrapper__(callback).__disown__() + def __arguments_to_numpy__(i, arg): assert isinstance(arg, swig_paddle.Arguments) value = arg.getSlotValue(i) + ids = arg.getSlotIds(i) if value is not None: assert isinstance(value, swig_paddle.Matrix) value = value.copyToNumpyMat() - ids = arg.getSlotIds(i) if ids is not None: assert isinstance(ids, swig_paddle.IVector) ids = ids.copyToNumpyArray() - return { - "value": value, - "id": ids - } + return {"value": value, "id": ids} + def __monkeypatch_gradient_machine__(): """ @@ -102,7 +100,6 @@ def __monkeypatch_gradient_machine__(): swig_paddle.GradientMachine.loadFromConfigFile = \ staticmethod(loadGradientMachine) - def __matrix_to_numpy__(m): if isinstance(m, swig_paddle.Matrix): return m.copyToNumpyMat() @@ -113,9 +110,11 @@ def __monkeypatch_gradient_machine__(): def createFromConfigProto(protoObj, createMode=swig_paddle.CREATE_MODE_NORMAL, - paramTypes=[swig_paddle.PARAMETER_VALUE, - swig_paddle.PARAMETER_GRADIENT, - swig_paddle.PARAMETER_MOMENTUM]): + paramTypes=[ + swig_paddle.PARAMETER_VALUE, + swig_paddle.PARAMETER_GRADIENT, + swig_paddle.PARAMETER_MOMENTUM + ]): """ Create Gradient Machine From Proto object. :param protoObj: Model config @@ -145,8 +144,10 @@ def __monkeypatch_gradient_machine__(): """ outArgs = swig_paddle.Arguments.createArguments(0) self.forward(inArgs, outArgs, swig_paddle.PASS_TEST) - return [__arguments_to_numpy__(i, outArgs) for i in xrange( - outArgs.getSlotNum())] + return [ + __arguments_to_numpy__(i, outArgs) + for i in xrange(outArgs.getSlotNum()) + ] swig_paddle.GradientMachine.forwardTest = forwardTest @@ -167,7 +168,10 @@ def __monkeypatch_gradient_machine__(): swig_paddle.GradientMachine.__forwardBackward__ = \ swig_paddle.GradientMachine.forwardBackward - def forwardBackward(self, inArgs, outArgs, passType, + def forwardBackward(self, + inArgs, + outArgs, + passType, callback=swig_paddle.UpdateCallback()): """ GradientMachine forward backward. @@ -315,9 +319,8 @@ class DataProviderWrapperConverter(object): self.cols += other def __call__(self, slot_idx, arg): - mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, - self.dim, - len(self.cols), True) + mat = swig_paddle.Matrix.createSparse( + len(self.indices) - 1, self.dim, len(self.cols), True) assert isinstance(mat, swig_paddle.Matrix) mat.sparseCopyFrom(self.indices, self.cols) self.putIntoArg(slot_idx, arg, mat) @@ -341,9 +344,8 @@ class DataProviderWrapperConverter(object): self.values += map(lambda x: x[1], other) def __call__(self, slot_idx, arg): - mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, - self.dim, - len(self.cols), False) + mat = swig_paddle.Matrix.createSparse( + len(self.indices) - 1, self.dim, len(self.cols), False) assert isinstance(mat, swig_paddle.Matrix) mat.sparseCopyFrom(self.indices, self.cols, self.values) self.putIntoArg(slot_idx, arg, mat) @@ -352,8 +354,9 @@ class DataProviderWrapperConverter(object): paddle.trainer.PyDataProviderWrapper.DenseSlot: DenseValueConverter, paddle.trainer.PyDataProviderWrapper.IndexSlot: IdValueConverter, paddle.trainer.PyDataProviderWrapper.SparseNonValueSlot: - SparseNonValueConverter, - paddle.trainer.PyDataProviderWrapper.SparseValueSlot: SparseValueConverter + SparseNonValueConverter, + paddle.trainer.PyDataProviderWrapper.SparseValueSlot: + SparseValueConverter } def __init__(self, use_seq, header): @@ -381,10 +384,9 @@ class DataProviderWrapperConverter(object): assert isinstance(argument, swig_paddle.Arguments) argument.resize(len(self.__header__)) - values = map(lambda x: - DataProviderWrapperConverter.__SLOT_VALUE_CONVERTER_MAP__[ - x.__class__](x), - self.__header__) + values = map( + lambda x: DataProviderWrapperConverter.__SLOT_VALUE_CONVERTER_MAP__[x.__class__](x), + self.__header__) if self.__use_seq__: seq_dim = [[] for _ in xrange(self.__header__.__len__())] @@ -394,14 +396,13 @@ class DataProviderWrapperConverter(object): for slot_idx, sequence in enumerate(each_sample): for raw_data in sequence: values[slot_idx].append(raw_data) - seq_start_pos[slot_idx].append( - seq_start_pos[slot_idx][-1] + len(sequence)) + seq_start_pos[slot_idx].append(seq_start_pos[slot_idx][-1] + + len(sequence)) seq_dim[slot_idx].append(len(sequence)) for slot_idx in xrange(len(self.__header__)): - argument.setSlotSequenceDim(slot_idx, - swig_paddle.IVector.create( - seq_dim[slot_idx])) + argument.setSlotSequenceDim( + slot_idx, swig_paddle.IVector.create(seq_dim[slot_idx])) argument.setSlotSequenceStartPositions( slot_idx, swig_paddle.IVector.create(seq_start_pos[slot_idx])) @@ -422,7 +423,6 @@ class DataProviderWrapperConverter(object): return self.convert(wrapper_data, argument) - def __monkey_patch_protobuf_objects__(): def ParameterConfig_toProto(self): """ @@ -459,8 +459,7 @@ def __monkey_patch_protobuf_objects__(): :return: paddle.OptimizationConfig """ - assert isinstance(protoObj, - paddle.proto.OptimizationConfig) + assert isinstance(protoObj, paddle.proto.OptimizationConfig) return swig_paddle.OptimizationConfig.createFromProtoString( protoObj.SerializeToString()) @@ -475,8 +474,7 @@ def __monkey_patch_protobuf_objects__(): :param protoObj: proto.TrainerConfig :return: paddle.TrainerConfig """ - assert isinstance(protoObj, - paddle.proto.TrainerConfig) + assert isinstance(protoObj, paddle.proto.TrainerConfig) return swig_paddle.TrainerConfig.createFromProtoString( protoObj.SerializeToString()) @@ -537,6 +535,7 @@ def __monkey_patch_trainer__(): assert isinstance(model, swig_paddle.GradientMachine) return swig_paddle.Trainer.__create__( swig_paddle.TrainerConfig.createFromProto(config), model) + swig_paddle.Trainer.create = staticmethod(Trainer_create) swig_paddle.Trainer.__getForwardOutput__ = \ @@ -551,14 +550,19 @@ def __monkey_patch_trainer__(): numpy.ndarray. """ outArgs = self.__getForwardOutput__() - return [__arguments_to_numpy__(i, outArgs) for i in xrange( - outArgs.getSlotNum())] + return [ + __arguments_to_numpy__(i, outArgs) + for i in xrange(outArgs.getSlotNum()) + ] swig_paddle.Trainer.getForwardOutput = getForwardOutput + def monkeypatches(): - patches = [__monkeypatch_init_paddle__, __monkeypatch_gradient_machine__, - __monkey_patch_protobuf_objects__, - __monkey_patch_parameter__, __monkey_patch_trainer__] + patches = [ + __monkeypatch_init_paddle__, __monkeypatch_gradient_machine__, + __monkey_patch_protobuf_objects__, __monkey_patch_parameter__, + __monkey_patch_trainer__ + ] for patch in patches: patch() diff --git a/paddle/scripts/CMakeLists.txt b/paddle/scripts/CMakeLists.txt index dee46055c5a4db8c77b4248de70573c02a60e631..1bae396a18688cd53e164774df07660ccc2451d7 100644 --- a/paddle/scripts/CMakeLists.txt +++ b/paddle/scripts/CMakeLists.txt @@ -6,4 +6,4 @@ configure_file(submit_local.sh.in install(FILES ${CMAKE_CURRENT_BINARY_DIR}/submit_local.sh DESTINATION bin PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ - RENAME paddle) \ No newline at end of file + RENAME paddle) diff --git a/paddle/scripts/cluster_train/conf.py b/paddle/scripts/cluster_train/conf.py index c8fd360e7552ed7c0f11aaa06574a11344c44aba..f1114a59201b9e57a14b739a327b622327c515f7 100644 --- a/paddle/scripts/cluster_train/conf.py +++ b/paddle/scripts/cluster_train/conf.py @@ -13,17 +13,14 @@ # limitations under the License. HOSTS = [ - "root@192.168.100.17", - "root@192.168.100.18", - ] - + "root@192.168.100.17", + "root@192.168.100.18", +] ''' workspace configuration ''' #root dir for workspace, can be set as any director with real user account ROOT_DIR = "/home/paddle" - - ''' network configuration ''' @@ -37,4 +34,4 @@ PADDLE_PORTS_NUM = 2 PADDLE_PORTS_NUM_FOR_SPARSE = 2 #environments setting for all processes in cluster job -LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/lib64" +LD_LIBRARY_PATH = "/usr/local/cuda/lib64:/usr/lib64" diff --git a/paddle/scripts/cluster_train/paddle.py b/paddle/scripts/cluster_train/paddle.py index 79698c72e619fa48c42d91d41abab61e2a5902ee..7343a600c1bf5522ac8b0cd90a38f8a362ba7ae6 100644 --- a/paddle/scripts/cluster_train/paddle.py +++ b/paddle/scripts/cluster_train/paddle.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - """ module for launching cluster job """ import os @@ -23,13 +21,13 @@ import copy import time import signal - from fabric.api import run, put, settings, env, prefix from fabric.tasks import execute #configuration for cluster import conf + def refine_unknown_args(cmd_args): ''' refine unknown parameters to handle some special parameters @@ -37,7 +35,7 @@ def refine_unknown_args(cmd_args): new_args = [] for arg in cmd_args: if arg.startswith("--") and arg.find("=") != -1: - equal_pos = arg.find("=") #find first = pos + equal_pos = arg.find("=") #find first = pos arglist = list(arg) arglist[equal_pos] = " " arg = "".join(arglist) @@ -50,6 +48,7 @@ def refine_unknown_args(cmd_args): new_args.append(arg) return new_args + def kill_process(): ''' kill comments threads @@ -60,6 +59,7 @@ def kill_process(): | awk '{print $2}' \ | xargs kill > /dev/null 2>&1") + def job_prepare(jobdir, data=None): ''' prepare job related workspace data @@ -70,6 +70,7 @@ def job_prepare(jobdir, data=None): This function just prepare all related model and other resources needed at runtime. ''' + def job_create_workspace(jobdir, data=None): ''' prepare job workspace, common file, etc. @@ -94,7 +95,8 @@ def job_prepare(jobdir, data=None): execute(set_nodefile, i, hosts=conf.HOSTS[i]) #clean rubbish caused by exception with settings(warn_only=True): - execute(kill_process, hosts=conf.HOSTS) + execute(kill_process, hosts=conf.HOSTS) + def job_pserver(jobdir, pids=None): ''' @@ -124,9 +126,8 @@ def job_pserver(jobdir, pids=None): execute(start_pserver, jobdir, pargs, hosts=conf.HOSTS) -def job_trainer(jobdir, - train_args_dict, - pids=None): + +def job_trainer(jobdir, train_args_dict, pids=None): ''' start paddle trainer ''' @@ -171,9 +172,8 @@ def job_trainer(jobdir, train_args += " --trainer_id=" + str(i) execute(start_trainer, jobdir, train_args, hosts=conf.HOSTS[i]) -def job_all(job_package, - jobdir=None, - train_args_dict=None): + +def job_all(job_package, jobdir=None, train_args_dict=None): ''' param job_package param train_args_dict @@ -183,41 +183,52 @@ def job_all(job_package, jobdir = conf.ROOT_DIR + "/JOB" + timestamp job_prepare(jobdir, job_package) job_pserver(jobdir) - time.sleep(5) #wait until pservers completely start + time.sleep(5) #wait until pservers completely start job_trainer(jobdir, train_args_dict) job_clean() + def job_clean(): ''' if starting job failed from paddle internal, the framework always is launched successfully since these process are daemon processes. so this job_clean can alway clean job rubbish process with ctrl+c. ''' + def signal_handler(signal, frame): ''' SIGINT handler ''' + def kill_process(): - run("ps aux \ + run("ps aux \ | grep paddle_process_by_paddle \ | grep -v grep \ | awk '{print $2}' \ | xargs kill > /dev/null 2>&1") + with settings(warn_only=True): - execute(kill_process, hosts=conf.HOSTS) + execute(kill_process, hosts=conf.HOSTS) signal.signal(signal.SIGINT, signal_handler) signal.pause() + if __name__ == '__main__': - parser = argparse.ArgumentParser(prog="paddle.py", - description='simple tool for cluster training') - parser.add_argument('-j', '--job_workspace', - required=False, default=None, - help='job workspace') - parser.add_argument('-p', '--job_dispatch_package', - required=False, default=None, - help='job package for dispatching to all other nodes') + parser = argparse.ArgumentParser( + prog="paddle.py", description='simple tool for cluster training') + parser.add_argument( + '-j', + '--job_workspace', + required=False, + default=None, + help='job workspace') + parser.add_argument( + '-p', + '--job_dispatch_package', + required=False, + default=None, + help='job package for dispatching to all other nodes') args, train_args_list = parser.parse_known_args() train_args = refine_unknown_args(train_args_list) @@ -227,14 +238,10 @@ if __name__ == '__main__': #if assigned workspace, do not need to dispatch data, #so job_local_package should be None assert args.job_dispatch_package is None - job_all(None, - args.job_workspace, - train_args_dict) + job_all(None, args.job_workspace, train_args_dict) elif args.job_dispatch_package is not None: assert args.job_workspace is None assert os.path.isdir(args.job_dispatch_package) - job_all(args.job_dispatch_package, - None, - train_args_dict) + job_all(args.job_dispatch_package, None, train_args_dict) else: print "--job_workspace or --job_dispatch_package should be set" diff --git a/paddle/scripts/cpplint.py b/paddle/scripts/cpplint.py index 5e905b865fc5167a2bddcf4ca0ab8313d17af4a3..157ce7b44ac3cfe3a8ca5eda78e959cf7be4cc5b 100644 --- a/paddle/scripts/cpplint.py +++ b/paddle/scripts/cpplint.py @@ -27,7 +27,6 @@ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """Does google-lint on c++ files. The goal of this script is to identify places in the code that *may* @@ -55,7 +54,6 @@ import string import sys import unicodedata - _USAGE = """ Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] [--counting=total|toplevel|detailed] [--root=subdir] @@ -242,13 +240,11 @@ _ERROR_CATEGORIES = [ 'whitespace/semicolon', 'whitespace/tab', 'whitespace/todo', - ] +] # These error categories are no longer enforced by cpplint, but for backwards- # compatibility they may still appear in NOLINT comments. -_LEGACY_ERROR_CATEGORIES = [ - 'readability/streams', - ] +_LEGACY_ERROR_CATEGORIES = ['readability/streams', ] # The default state of the category filter. This is overridden by the --filter= # flag. By default all errors are on, so only add here categories that should be @@ -394,8 +390,7 @@ _CPP_HEADERS = frozenset([ 'cuchar', 'cwchar', 'cwctype', - ]) - +]) # These headers are excluded from [build/include] and [build/include_order] # checks: @@ -405,38 +400,40 @@ _CPP_HEADERS = frozenset([ _THIRD_PARTY_HEADERS_PATTERN = re.compile( r'^(?:[^/]*[A-Z][^/]*\.h|lua\.h|lauxlib\.h|lualib\.h)$') - # Assertion macros. These are defined in base/logging.h and # testing/base/gunit.h. Note that the _M versions need to come first # for substring matching to work. _CHECK_MACROS = [ - 'DCHECK', 'CHECK', - 'EXPECT_TRUE_M', 'EXPECT_TRUE', - 'ASSERT_TRUE_M', 'ASSERT_TRUE', - 'EXPECT_FALSE_M', 'EXPECT_FALSE', - 'ASSERT_FALSE_M', 'ASSERT_FALSE', - ] + 'DCHECK', + 'CHECK', + 'EXPECT_TRUE_M', + 'EXPECT_TRUE', + 'ASSERT_TRUE_M', + 'ASSERT_TRUE', + 'EXPECT_FALSE_M', + 'EXPECT_FALSE', + 'ASSERT_FALSE_M', + 'ASSERT_FALSE', +] # Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE _CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS]) -for op, replacement in [('==', 'EQ'), ('!=', 'NE'), - ('>=', 'GE'), ('>', 'GT'), +for op, replacement in [('==', 'EQ'), ('!=', 'NE'), ('>=', 'GE'), ('>', 'GT'), ('<=', 'LE'), ('<', 'LT')]: - _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement - _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement - _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement - _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement - _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement - _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement - -for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), - ('>=', 'LT'), ('>', 'LE'), - ('<=', 'GT'), ('<', 'GE')]: - _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement - _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement - _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement - _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement + _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement + _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement + +for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), ('>=', 'LT'), + ('>', 'LE'), ('<=', 'GT'), ('<', 'GE')]: + _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement + _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement # Alternative tokens and their replacements. For full list, see section 2.5 # Alternative tokens [lex.digraph] in the C++ standard. @@ -455,16 +452,15 @@ _ALT_TOKEN_REPLACEMENT = { 'xor_eq': '^=', 'not': '!', 'not_eq': '!=' - } +} # Compile regular expression that matches all the above keywords. The "[ =()]" # bit is meant to avoid matching these keywords outside of boolean expressions. # # False positives include C-style multi-line comments and multi-line strings # but those have always been troublesome for cpplint. -_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile( - r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') - +_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile(r'[ =()](' + ('|'.join( + _ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)') # These constants define types of headers for use with # _IncludeState.CheckNextIncludeOrder(). @@ -475,17 +471,16 @@ _POSSIBLE_MY_HEADER = 4 _OTHER_HEADER = 5 # These constants define the current inline assembly state -_NO_ASM = 0 # Outside of inline assembly block -_INSIDE_ASM = 1 # Inside inline assembly block -_END_ASM = 2 # Last line of inline assembly block -_BLOCK_ASM = 3 # The whole block is an inline assembly block +_NO_ASM = 0 # Outside of inline assembly block +_INSIDE_ASM = 1 # Inside inline assembly block +_END_ASM = 2 # Last line of inline assembly block +_BLOCK_ASM = 3 # The whole block is an inline assembly block # Match start of assembly blocks _MATCH_ASM = re.compile(r'^\s*(?:asm|_asm|__asm|__asm__)' r'(?:\s+(volatile|__volatile__))?' r'\s*[{(]') - _regexp_compile_cache = {} # {str, set(int)}: a map from error categories to sets of linenumbers @@ -504,8 +499,9 @@ _line_length = 80 # This is set by --extensions flag. _valid_extensions = set(['cc', 'h', 'cpp', 'cu', 'cuh']) + def ParseNolintSuppressions(filename, raw_line, linenum, error): - """Updates the global list of error-suppressions. + """Updates the global list of error-suppressions. Parses any NOLINT comments on the current line, updating the global error_suppressions store. Reports an error if the NOLINT comment @@ -517,45 +513,47 @@ def ParseNolintSuppressions(filename, raw_line, linenum, error): linenum: int, the number of the current line. error: function, an error handler. """ - matched = Search(r'\bNOLINT(NEXTLINE(S_\d+)?)?\b(\([^)]+\))?', raw_line) - if matched: - if matched.group(1): - lines = matched.group(2) - if lines : - lines=int(lines[2:]) - suppressed_line = [ linenum + i for i in xrange(lines) ] - else: - suppressed_line = linenum + 1 - else: - suppressed_line = linenum - category = matched.group(3) - if category in (None, '(*)'): # => "suppress all" - if isinstance(suppressed_line, int): - _error_suppressions.setdefault(None, set()).add(suppressed_line) - else: - for _line in suppressed_line: - _error_suppressions.setdefault(None, set()).add(_line) - else: - if category.startswith('(') and category.endswith(')'): - category = category[1:-1] - if category in _ERROR_CATEGORIES: - if isinstance(suppressed_line, int): - _error_suppressions.setdefault(category, set()).add(suppressed_line) - else: - for _line in suppressed_line: - _error_suppressions.setdefault(category, set()).add(_line) - elif category not in _LEGACY_ERROR_CATEGORIES: - error(filename, linenum, 'readability/nolint', 5, - 'Unknown NOLINT error category: %s' % category) + matched = Search(r'\bNOLINT(NEXTLINE(S_\d+)?)?\b(\([^)]+\))?', raw_line) + if matched: + if matched.group(1): + lines = matched.group(2) + if lines: + lines = int(lines[2:]) + suppressed_line = [linenum + i for i in xrange(lines)] + else: + suppressed_line = linenum + 1 + else: + suppressed_line = linenum + category = matched.group(3) + if category in (None, '(*)'): # => "suppress all" + if isinstance(suppressed_line, int): + _error_suppressions.setdefault(None, set()).add(suppressed_line) + else: + for _line in suppressed_line: + _error_suppressions.setdefault(None, set()).add(_line) + else: + if category.startswith('(') and category.endswith(')'): + category = category[1:-1] + if category in _ERROR_CATEGORIES: + if isinstance(suppressed_line, int): + _error_suppressions.setdefault( + category, set()).add(suppressed_line) + else: + for _line in suppressed_line: + _error_suppressions.setdefault(category, + set()).add(_line) + elif category not in _LEGACY_ERROR_CATEGORIES: + error(filename, linenum, 'readability/nolint', 5, + 'Unknown NOLINT error category: %s' % category) def ResetNolintSuppressions(): - """Resets the set of NOLINT suppressions to empty.""" - _error_suppressions.clear() + """Resets the set of NOLINT suppressions to empty.""" + _error_suppressions.clear() def IsErrorSuppressedByNolint(category, linenum): - """Returns true if the specified error category is suppressed on this line. + """Returns true if the specified error category is suppressed on this line. Consults the global error_suppressions map populated by ParseNolintSuppressions/ResetNolintSuppressions. @@ -566,22 +564,22 @@ def IsErrorSuppressedByNolint(category, linenum): Returns: bool, True iff the error should be suppressed due to a NOLINT comment. """ - return (linenum in _error_suppressions.get(category, set()) or - linenum in _error_suppressions.get(None, set())) + return (linenum in _error_suppressions.get(category, set()) or + linenum in _error_suppressions.get(None, set())) def Match(pattern, s): - """Matches the string with the pattern, caching the compiled regexp.""" - # The regexp compilation caching is inlined in both Match and Search for - # performance reasons; factoring it out into a separate function turns out - # to be noticeably expensive. - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].match(s) + """Matches the string with the pattern, caching the compiled regexp.""" + # The regexp compilation caching is inlined in both Match and Search for + # performance reasons; factoring it out into a separate function turns out + # to be noticeably expensive. + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].match(s) def ReplaceAll(pattern, rep, s): - """Replaces instances of pattern in a string with a replacement. + """Replaces instances of pattern in a string with a replacement. The compiled regex is kept in a cache shared by Match and Search. @@ -593,20 +591,20 @@ def ReplaceAll(pattern, rep, s): Returns: string with replacements made (or original string if no replacements) """ - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].sub(rep, s) + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].sub(rep, s) def Search(pattern, s): - """Searches the string for the pattern, caching the compiled regexp.""" - if pattern not in _regexp_compile_cache: - _regexp_compile_cache[pattern] = sre_compile.compile(pattern) - return _regexp_compile_cache[pattern].search(s) + """Searches the string for the pattern, caching the compiled regexp.""" + if pattern not in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].search(s) class _IncludeState(object): - """Tracks line numbers for includes, and the order in which includes appear. + """Tracks line numbers for includes, and the order in which includes appear. include_list contains list of lists of (header, line number) pairs. It's a lists of lists rather than just one flat list to make it @@ -617,35 +615,35 @@ class _IncludeState(object): raise an _IncludeError with an appropriate error message. """ - # self._section will move monotonically through this set. If it ever - # needs to move backwards, CheckNextIncludeOrder will raise an error. - _INITIAL_SECTION = 0 - _MY_H_SECTION = 1 - _C_SECTION = 2 - _CPP_SECTION = 3 - _OTHER_H_SECTION = 4 - - _TYPE_NAMES = { - _C_SYS_HEADER: 'C system header', - _CPP_SYS_HEADER: 'C++ system header', - _LIKELY_MY_HEADER: 'header this file implements', - _POSSIBLE_MY_HEADER: 'header this file may implement', - _OTHER_HEADER: 'other header', - } - _SECTION_NAMES = { - _INITIAL_SECTION: "... nothing. (This can't be an error.)", - _MY_H_SECTION: 'a header this file implements', - _C_SECTION: 'C system header', - _CPP_SECTION: 'C++ system header', - _OTHER_H_SECTION: 'other header', - } - - def __init__(self): - self.include_list = [[]] - self.ResetSection('') - - def FindHeader(self, header): - """Check if a header has already been included. + # self._section will move monotonically through this set. If it ever + # needs to move backwards, CheckNextIncludeOrder will raise an error. + _INITIAL_SECTION = 0 + _MY_H_SECTION = 1 + _C_SECTION = 2 + _CPP_SECTION = 3 + _OTHER_H_SECTION = 4 + + _TYPE_NAMES = { + _C_SYS_HEADER: 'C system header', + _CPP_SYS_HEADER: 'C++ system header', + _LIKELY_MY_HEADER: 'header this file implements', + _POSSIBLE_MY_HEADER: 'header this file may implement', + _OTHER_HEADER: 'other header', + } + _SECTION_NAMES = { + _INITIAL_SECTION: "... nothing. (This can't be an error.)", + _MY_H_SECTION: 'a header this file implements', + _C_SECTION: 'C system header', + _CPP_SECTION: 'C++ system header', + _OTHER_H_SECTION: 'other header', + } + + def __init__(self): + self.include_list = [[]] + self.ResetSection('') + + def FindHeader(self, header): + """Check if a header has already been included. Args: header: header to check. @@ -653,35 +651,35 @@ class _IncludeState(object): Line number of previous occurrence, or -1 if the header has not been seen before. """ - for section_list in self.include_list: - for f in section_list: - if f[0] == header: - return f[1] - return -1 + for section_list in self.include_list: + for f in section_list: + if f[0] == header: + return f[1] + return -1 - def ResetSection(self, directive): - """Reset section checking for preprocessor directive. + def ResetSection(self, directive): + """Reset section checking for preprocessor directive. Args: directive: preprocessor directive (e.g. "if", "else"). """ - # The name of the current section. - self._section = self._INITIAL_SECTION - # The path of last found header. - self._last_header = '' + # The name of the current section. + self._section = self._INITIAL_SECTION + # The path of last found header. + self._last_header = '' - # Update list of includes. Note that we never pop from the - # include list. - if directive in ('if', 'ifdef', 'ifndef'): - self.include_list.append([]) - elif directive in ('else', 'elif'): - self.include_list[-1] = [] + # Update list of includes. Note that we never pop from the + # include list. + if directive in ('if', 'ifdef', 'ifndef'): + self.include_list.append([]) + elif directive in ('else', 'elif'): + self.include_list[-1] = [] - def SetLastHeader(self, header_path): - self._last_header = header_path + def SetLastHeader(self, header_path): + self._last_header = header_path - def CanonicalizeAlphabeticalOrder(self, header_path): - """Returns a path canonicalized for alphabetical comparison. + def CanonicalizeAlphabeticalOrder(self, header_path): + """Returns a path canonicalized for alphabetical comparison. - replaces "-" with "_" so they both cmp the same. - removes '-inl' since we don't require them to be after the main header. @@ -693,10 +691,10 @@ class _IncludeState(object): Returns: Canonicalized path. """ - return header_path.replace('-inl.h', '.h').replace('-', '_').lower() + return header_path.replace('-inl.h', '.h').replace('-', '_').lower() - def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): - """Check if a header is in alphabetical order with the previous header. + def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path): + """Check if a header is in alphabetical order with the previous header. Args: clean_lines: A CleansedLines instance containing the file. @@ -706,18 +704,18 @@ class _IncludeState(object): Returns: Returns true if the header is in alphabetical order. """ - # If previous section is different from current section, _last_header will - # be reset to empty string, so it's always less than current header. - # - # If previous line was a blank line, assume that the headers are - # intentionally sorted the way they are. - if (self._last_header > header_path and - Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): - return False - return True + # If previous section is different from current section, _last_header will + # be reset to empty string, so it's always less than current header. + # + # If previous line was a blank line, assume that the headers are + # intentionally sorted the way they are. + if (self._last_header > header_path and + Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])): + return False + return True - def CheckNextIncludeOrder(self, header_type): - """Returns a non-empty error message if the next header is out of order. + def CheckNextIncludeOrder(self, header_type): + """Returns a non-empty error message if the next header is out of order. This function also updates the internal state to be ready to check the next include. @@ -730,80 +728,79 @@ class _IncludeState(object): error message describing what's wrong. """ - error_message = ('Found %s after %s' % - (self._TYPE_NAMES[header_type], - self._SECTION_NAMES[self._section])) - - last_section = self._section - - if header_type == _C_SYS_HEADER: - if self._section <= self._C_SECTION: - self._section = self._C_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _CPP_SYS_HEADER: - if self._section <= self._CPP_SECTION: - self._section = self._CPP_SECTION - else: - self._last_header = '' - return error_message - elif header_type == _LIKELY_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - self._section = self._OTHER_H_SECTION - elif header_type == _POSSIBLE_MY_HEADER: - if self._section <= self._MY_H_SECTION: - self._section = self._MY_H_SECTION - else: - # This will always be the fallback because we're not sure - # enough that the header is associated with this file. - self._section = self._OTHER_H_SECTION - else: - assert header_type == _OTHER_HEADER - self._section = self._OTHER_H_SECTION + error_message = ('Found %s after %s' % ( + self._TYPE_NAMES[header_type], self._SECTION_NAMES[self._section])) + + last_section = self._section + + if header_type == _C_SYS_HEADER: + if self._section <= self._C_SECTION: + self._section = self._C_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _CPP_SYS_HEADER: + if self._section <= self._CPP_SECTION: + self._section = self._CPP_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _LIKELY_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + self._section = self._OTHER_H_SECTION + elif header_type == _POSSIBLE_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + # This will always be the fallback because we're not sure + # enough that the header is associated with this file. + self._section = self._OTHER_H_SECTION + else: + assert header_type == _OTHER_HEADER + self._section = self._OTHER_H_SECTION - if last_section != self._section: - self._last_header = '' + if last_section != self._section: + self._last_header = '' - return '' + return '' class _CppLintState(object): - """Maintains module-wide state..""" - - def __init__(self): - self.verbose_level = 1 # global setting. - self.error_count = 0 # global count of reported errors - # filters to apply when emitting error messages - self.filters = _DEFAULT_FILTERS[:] - # backup of filter list. Used to restore the state after each file. - self._filters_backup = self.filters[:] - self.counting = 'total' # In what way are we counting errors? - self.errors_by_category = {} # string to int dict storing error counts - - # output format: - # "emacs" - format that emacs can parse (default) - # "vs7" - format that Microsoft Visual Studio 7 can parse - self.output_format = 'emacs' - - def SetOutputFormat(self, output_format): - """Sets the output format for errors.""" - self.output_format = output_format - - def SetVerboseLevel(self, level): - """Sets the module's verbosity, and returns the previous setting.""" - last_verbose_level = self.verbose_level - self.verbose_level = level - return last_verbose_level - - def SetCountingStyle(self, counting_style): - """Sets the module's counting options.""" - self.counting = counting_style - - def SetFilters(self, filters): - """Sets the error-message filters. + """Maintains module-wide state..""" + + def __init__(self): + self.verbose_level = 1 # global setting. + self.error_count = 0 # global count of reported errors + # filters to apply when emitting error messages + self.filters = _DEFAULT_FILTERS[:] + # backup of filter list. Used to restore the state after each file. + self._filters_backup = self.filters[:] + self.counting = 'total' # In what way are we counting errors? + self.errors_by_category = {} # string to int dict storing error counts + + # output format: + # "emacs" - format that emacs can parse (default) + # "vs7" - format that Microsoft Visual Studio 7 can parse + self.output_format = 'emacs' + + def SetOutputFormat(self, output_format): + """Sets the output format for errors.""" + self.output_format = output_format + + def SetVerboseLevel(self, level): + """Sets the module's verbosity, and returns the previous setting.""" + last_verbose_level = self.verbose_level + self.verbose_level = level + return last_verbose_level + + def SetCountingStyle(self, counting_style): + """Sets the module's counting options.""" + self.counting = counting_style + + def SetFilters(self, filters): + """Sets the error-message filters. These filters are applied when deciding whether to emit a given error message. @@ -816,86 +813,88 @@ class _CppLintState(object): ValueError: The comma-separated filters did not all start with '+' or '-'. E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" """ - # Default filters always have less priority than the flag ones. - self.filters = _DEFAULT_FILTERS[:] - self.AddFilters(filters) - - def AddFilters(self, filters): - """ Adds more filters to the existing list of error-message filters. """ - for filt in filters.split(','): - clean_filt = filt.strip() - if clean_filt: - self.filters.append(clean_filt) - for filt in self.filters: - if not (filt.startswith('+') or filt.startswith('-')): - raise ValueError('Every filter in --filters must start with + or -' - ' (%s does not)' % filt) - - def BackupFilters(self): - """ Saves the current filter list to backup storage.""" - self._filters_backup = self.filters[:] + # Default filters always have less priority than the flag ones. + self.filters = _DEFAULT_FILTERS[:] + self.AddFilters(filters) + + def AddFilters(self, filters): + """ Adds more filters to the existing list of error-message filters. """ + for filt in filters.split(','): + clean_filt = filt.strip() + if clean_filt: + self.filters.append(clean_filt) + for filt in self.filters: + if not (filt.startswith('+') or filt.startswith('-')): + raise ValueError( + 'Every filter in --filters must start with + or -' + ' (%s does not)' % filt) + + def BackupFilters(self): + """ Saves the current filter list to backup storage.""" + self._filters_backup = self.filters[:] + + def RestoreFilters(self): + """ Restores filters previously backed up.""" + self.filters = self._filters_backup[:] + + def ResetErrorCounts(self): + """Sets the module's error statistic back to zero.""" + self.error_count = 0 + self.errors_by_category = {} + + def IncrementErrorCount(self, category): + """Bumps the module's error statistic.""" + self.error_count += 1 + if self.counting in ('toplevel', 'detailed'): + if self.counting != 'detailed': + category = category.split('/')[0] + if category not in self.errors_by_category: + self.errors_by_category[category] = 0 + self.errors_by_category[category] += 1 + + def PrintErrorCounts(self): + """Print a summary of errors by category, and the total.""" + for category, count in self.errors_by_category.iteritems(): + sys.stdout.write('Category \'%s\' errors found: %d\n' % + (category, count)) + sys.stdout.write('Total errors found: %d\n' % self.error_count) - def RestoreFilters(self): - """ Restores filters previously backed up.""" - self.filters = self._filters_backup[:] - - def ResetErrorCounts(self): - """Sets the module's error statistic back to zero.""" - self.error_count = 0 - self.errors_by_category = {} - - def IncrementErrorCount(self, category): - """Bumps the module's error statistic.""" - self.error_count += 1 - if self.counting in ('toplevel', 'detailed'): - if self.counting != 'detailed': - category = category.split('/')[0] - if category not in self.errors_by_category: - self.errors_by_category[category] = 0 - self.errors_by_category[category] += 1 - - def PrintErrorCounts(self): - """Print a summary of errors by category, and the total.""" - for category, count in self.errors_by_category.iteritems(): - sys.stdout.write('Category \'%s\' errors found: %d\n' % - (category, count)) - sys.stdout.write('Total errors found: %d\n' % self.error_count) _cpplint_state = _CppLintState() def _OutputFormat(): - """Gets the module's output format.""" - return _cpplint_state.output_format + """Gets the module's output format.""" + return _cpplint_state.output_format def _SetOutputFormat(output_format): - """Sets the module's output format.""" - _cpplint_state.SetOutputFormat(output_format) + """Sets the module's output format.""" + _cpplint_state.SetOutputFormat(output_format) def _VerboseLevel(): - """Returns the module's verbosity setting.""" - return _cpplint_state.verbose_level + """Returns the module's verbosity setting.""" + return _cpplint_state.verbose_level def _SetVerboseLevel(level): - """Sets the module's verbosity, and returns the previous setting.""" - return _cpplint_state.SetVerboseLevel(level) + """Sets the module's verbosity, and returns the previous setting.""" + return _cpplint_state.SetVerboseLevel(level) def _SetCountingStyle(level): - """Sets the module's counting options.""" - _cpplint_state.SetCountingStyle(level) + """Sets the module's counting options.""" + _cpplint_state.SetCountingStyle(level) def _Filters(): - """Returns the module's list of output filters, as a list.""" - return _cpplint_state.filters + """Returns the module's list of output filters, as a list.""" + return _cpplint_state.filters def _SetFilters(filters): - """Sets the module's error-message filters. + """Sets the module's error-message filters. These filters are applied when deciding whether to emit a given error message. @@ -904,10 +903,11 @@ def _SetFilters(filters): filters: A string of comma-separated filters (eg "whitespace/indent"). Each filter should start with + or -; else we die. """ - _cpplint_state.SetFilters(filters) + _cpplint_state.SetFilters(filters) + def _AddFilters(filters): - """Adds more filter overrides. + """Adds more filter overrides. Unlike _SetFilters, this function does not reset the current list of filters available. @@ -916,93 +916,97 @@ def _AddFilters(filters): filters: A string of comma-separated filters (eg "whitespace/indent"). Each filter should start with + or -; else we die. """ - _cpplint_state.AddFilters(filters) + _cpplint_state.AddFilters(filters) + def _BackupFilters(): - """ Saves the current filter list to backup storage.""" - _cpplint_state.BackupFilters() + """ Saves the current filter list to backup storage.""" + _cpplint_state.BackupFilters() + def _RestoreFilters(): - """ Restores filters previously backed up.""" - _cpplint_state.RestoreFilters() + """ Restores filters previously backed up.""" + _cpplint_state.RestoreFilters() + class _FunctionState(object): - """Tracks current function name and the number of lines in its body.""" + """Tracks current function name and the number of lines in its body.""" - _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. - _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. + _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. + _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. - def __init__(self): - self.in_a_function = False - self.lines_in_function = 0 - self.current_function = '' + def __init__(self): + self.in_a_function = False + self.lines_in_function = 0 + self.current_function = '' - def Begin(self, function_name): - """Start analyzing function body. + def Begin(self, function_name): + """Start analyzing function body. Args: function_name: The name of the function being tracked. """ - self.in_a_function = True - self.lines_in_function = 0 - self.current_function = function_name + self.in_a_function = True + self.lines_in_function = 0 + self.current_function = function_name - def Count(self): - """Count line in current function body.""" - if self.in_a_function: - self.lines_in_function += 1 + def Count(self): + """Count line in current function body.""" + if self.in_a_function: + self.lines_in_function += 1 - def Check(self, error, filename, linenum): - """Report if too many lines in function body. + def Check(self, error, filename, linenum): + """Report if too many lines in function body. Args: error: The function to call with any errors found. filename: The name of the current file. linenum: The number of the line to check. """ - if Match(r'T(EST|est)', self.current_function): - base_trigger = self._TEST_TRIGGER - else: - base_trigger = self._NORMAL_TRIGGER - trigger = base_trigger * 2**_VerboseLevel() - - if self.lines_in_function > trigger: - error_level = int(math.log(self.lines_in_function / base_trigger, 2)) - # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... - if error_level > 5: - error_level = 5 - error(filename, linenum, 'readability/fn_size', error_level, - 'Small and focused functions are preferred:' - ' %s has %d non-comment lines' - ' (error triggered by exceeding %d lines).' % ( - self.current_function, self.lines_in_function, trigger)) - - def End(self): - """Stop analyzing function body.""" - self.in_a_function = False + if Match(r'T(EST|est)', self.current_function): + base_trigger = self._TEST_TRIGGER + else: + base_trigger = self._NORMAL_TRIGGER + trigger = base_trigger * 2**_VerboseLevel() + + if self.lines_in_function > trigger: + error_level = int( + math.log(self.lines_in_function / base_trigger, 2)) + # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... + if error_level > 5: + error_level = 5 + error(filename, linenum, 'readability/fn_size', error_level, + 'Small and focused functions are preferred:' + ' %s has %d non-comment lines' + ' (error triggered by exceeding %d lines).' % ( + self.current_function, self.lines_in_function, trigger)) + + def End(self): + """Stop analyzing function body.""" + self.in_a_function = False class _IncludeError(Exception): - """Indicates a problem with the include order in a file.""" - pass + """Indicates a problem with the include order in a file.""" + pass class FileInfo(object): - """Provides utility functions for filenames. + """Provides utility functions for filenames. FileInfo provides easy access to the components of a file's path relative to the project root. """ - def __init__(self, filename): - self._filename = filename + def __init__(self, filename): + self._filename = filename - def FullName(self): - """Make Windows paths like Unix.""" - return os.path.abspath(self._filename).replace('\\', '/') + def FullName(self): + """Make Windows paths like Unix.""" + return os.path.abspath(self._filename).replace('\\', '/') - def RepositoryName(self): - """FullName after removing the local path to the repository. + def RepositoryName(self): + """FullName after removing the local path to the repository. If we have a real absolute path name here we can try to do something smart: detecting the root of the checkout and truncating /path/to/checkout from @@ -1011,43 +1015,43 @@ class FileInfo(object): people on different computers who have checked the source out to different locations won't see bogus errors. """ - fullname = self.FullName() - - if os.path.exists(fullname): - project_dir = os.path.dirname(fullname) - - if os.path.exists(os.path.join(project_dir, ".svn")): - # If there's a .svn file in the current directory, we recursively look - # up the directory tree for the top of the SVN checkout - root_dir = project_dir - one_up_dir = os.path.dirname(root_dir) - while os.path.exists(os.path.join(one_up_dir, ".svn")): - root_dir = os.path.dirname(root_dir) - one_up_dir = os.path.dirname(one_up_dir) - - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] - - # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by - # searching up from the current path. - root_dir = os.path.dirname(fullname) - while (root_dir != os.path.dirname(root_dir) and - not os.path.exists(os.path.join(root_dir, ".git")) and - not os.path.exists(os.path.join(root_dir, ".hg")) and - not os.path.exists(os.path.join(root_dir, ".svn"))): - root_dir = os.path.dirname(root_dir) - - if (os.path.exists(os.path.join(root_dir, ".git")) or - os.path.exists(os.path.join(root_dir, ".hg")) or - os.path.exists(os.path.join(root_dir, ".svn"))): - prefix = os.path.commonprefix([root_dir, project_dir]) - return fullname[len(prefix) + 1:] - - # Don't know what to do; header guard warnings may be wrong... - return fullname - - def Split(self): - """Splits the file into the directory, basename, and extension. + fullname = self.FullName() + + if os.path.exists(fullname): + project_dir = os.path.dirname(fullname) + + if os.path.exists(os.path.join(project_dir, ".svn")): + # If there's a .svn file in the current directory, we recursively look + # up the directory tree for the top of the SVN checkout + root_dir = project_dir + one_up_dir = os.path.dirname(root_dir) + while os.path.exists(os.path.join(one_up_dir, ".svn")): + root_dir = os.path.dirname(root_dir) + one_up_dir = os.path.dirname(one_up_dir) + + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by + # searching up from the current path. + root_dir = os.path.dirname(fullname) + while (root_dir != os.path.dirname(root_dir) and + not os.path.exists(os.path.join(root_dir, ".git")) and + not os.path.exists(os.path.join(root_dir, ".hg")) and + not os.path.exists(os.path.join(root_dir, ".svn"))): + root_dir = os.path.dirname(root_dir) + + if (os.path.exists(os.path.join(root_dir, ".git")) or + os.path.exists(os.path.join(root_dir, ".hg")) or + os.path.exists(os.path.join(root_dir, ".svn"))): + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Don't know what to do; header guard warnings may be wrong... + return fullname + + def Split(self): + """Splits the file into the directory, basename, and extension. For 'chrome/browser/browser.cc', Split() would return ('chrome/browser', 'browser', '.cc') @@ -1056,57 +1060,57 @@ class FileInfo(object): A tuple of (directory, basename, extension). """ - googlename = self.RepositoryName() - project, rest = os.path.split(googlename) - return (project,) + os.path.splitext(rest) + googlename = self.RepositoryName() + project, rest = os.path.split(googlename) + return (project, ) + os.path.splitext(rest) - def BaseName(self): - """File base name - text after the final slash, before the final period.""" - return self.Split()[1] + def BaseName(self): + """File base name - text after the final slash, before the final period.""" + return self.Split()[1] - def Extension(self): - """File extension - text following the final period.""" - return self.Split()[2] + def Extension(self): + """File extension - text following the final period.""" + return self.Split()[2] - def NoExtension(self): - """File has no source file extension.""" - return '/'.join(self.Split()[0:2]) + def NoExtension(self): + """File has no source file extension.""" + return '/'.join(self.Split()[0:2]) - def IsSource(self): - """File has a source file extension.""" - return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx') + def IsSource(self): + """File has a source file extension.""" + return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx') def _ShouldPrintError(category, confidence, linenum): - """If confidence >= verbose, category passes filter and is not suppressed.""" + """If confidence >= verbose, category passes filter and is not suppressed.""" - # There are three ways we might decide not to print an error message: - # a "NOLINT(category)" comment appears in the source, - # the verbosity level isn't high enough, or the filters filter it out. - if IsErrorSuppressedByNolint(category, linenum): - return False + # There are three ways we might decide not to print an error message: + # a "NOLINT(category)" comment appears in the source, + # the verbosity level isn't high enough, or the filters filter it out. + if IsErrorSuppressedByNolint(category, linenum): + return False - if confidence < _cpplint_state.verbose_level: - return False + if confidence < _cpplint_state.verbose_level: + return False - is_filtered = False - for one_filter in _Filters(): - if one_filter.startswith('-'): - if category.startswith(one_filter[1:]): - is_filtered = True - elif one_filter.startswith('+'): - if category.startswith(one_filter[1:]): - is_filtered = False - else: - assert False # should have been checked for in SetFilter. - if is_filtered: - return False + is_filtered = False + for one_filter in _Filters(): + if one_filter.startswith('-'): + if category.startswith(one_filter[1:]): + is_filtered = True + elif one_filter.startswith('+'): + if category.startswith(one_filter[1:]): + is_filtered = False + else: + assert False # should have been checked for in SetFilter. + if is_filtered: + return False - return True + return True def Error(filename, linenum, category, confidence, message): - """Logs the fact we've found a lint error. + """Logs the fact we've found a lint error. We log where the error was found, and also our confidence in the error, that is, how certain we are this is a legitimate style regression, and @@ -1127,17 +1131,17 @@ def Error(filename, linenum, category, confidence, message): and 1 meaning that it could be a legitimate construct. message: The error message. """ - if _ShouldPrintError(category, confidence, linenum): - _cpplint_state.IncrementErrorCount(category) - if _cpplint_state.output_format == 'vs7': - sys.stderr.write('%s(%s): %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) - elif _cpplint_state.output_format == 'eclipse': - sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) - else: - sys.stderr.write('%s:%s: %s [%s] [%d]\n' % ( - filename, linenum, message, category, confidence)) + if _ShouldPrintError(category, confidence, linenum): + _cpplint_state.IncrementErrorCount(category) + if _cpplint_state.output_format == 'vs7': + sys.stderr.write('%s(%s): %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) + elif _cpplint_state.output_format == 'eclipse': + sys.stderr.write('%s:%s: warning: %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) + else: + sys.stderr.write('%s:%s: %s [%s] [%d]\n' % + (filename, linenum, message, category, confidence)) # Matches standard C++ escape sequences per 2.13.2.3 of the C++ standard. @@ -1154,14 +1158,13 @@ _RE_PATTERN_C_COMMENTS = r'/\*(?:[^*]|\*(?!/))*\*/' # if this doesn't work we try on left side but only if there's a non-character # on the right. _RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( - r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + - _RE_PATTERN_C_COMMENTS + r'\s+|' + - r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + + r'(\s*' + _RE_PATTERN_C_COMMENTS + r'\s*$|' + _RE_PATTERN_C_COMMENTS + + r'\s+|' + r'\s+' + _RE_PATTERN_C_COMMENTS + r'(?=\W)|' + _RE_PATTERN_C_COMMENTS + r')') def IsCppString(line): - """Does line terminate so, that the next symbol is in string constant. + """Does line terminate so, that the next symbol is in string constant. This function does not consider single-line nor multi-line comments. @@ -1173,12 +1176,12 @@ def IsCppString(line): string constant. """ - line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" - return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 + line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" + return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 def CleanseRawStrings(raw_lines): - """Removes C++11 raw strings from lines. + """Removes C++11 raw strings from lines. Before: static const char kData[] = R"( @@ -1197,98 +1200,100 @@ def CleanseRawStrings(raw_lines): list of lines with C++11 raw strings replaced by empty strings. """ - delimiter = None - lines_without_raw_strings = [] - for line in raw_lines: - if delimiter: - # Inside a raw string, look for the end - end = line.find(delimiter) - if end >= 0: - # Found the end of the string, match leading space for this - # line and resume copying the original lines, and also insert - # a "" on the last line. - leading_space = Match(r'^(\s*)\S', line) - line = leading_space.group(1) + '""' + line[end + len(delimiter):] - delimiter = None - else: - # Haven't found the end yet, append a blank line. - line = '""' - - # Look for beginning of a raw string, and replace them with - # empty strings. This is done in a loop to handle multiple raw - # strings on the same line. - while delimiter is None: - # Look for beginning of a raw string. - # See 2.14.15 [lex.string] for syntax. - matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', line) - if matched: - delimiter = ')' + matched.group(2) + '"' - - end = matched.group(3).find(delimiter) - if end >= 0: - # Raw string ended on same line - line = (matched.group(1) + '""' + - matched.group(3)[end + len(delimiter):]) - delimiter = None - else: - # Start of a multi-line raw string - line = matched.group(1) + '""' - else: - break - - lines_without_raw_strings.append(line) - - # TODO(unknown): if delimiter is not None here, we might want to - # emit a warning for unterminated string. - return lines_without_raw_strings + delimiter = None + lines_without_raw_strings = [] + for line in raw_lines: + if delimiter: + # Inside a raw string, look for the end + end = line.find(delimiter) + if end >= 0: + # Found the end of the string, match leading space for this + # line and resume copying the original lines, and also insert + # a "" on the last line. + leading_space = Match(r'^(\s*)\S', line) + line = leading_space.group(1) + '""' + line[end + len( + delimiter):] + delimiter = None + else: + # Haven't found the end yet, append a blank line. + line = '""' + + # Look for beginning of a raw string, and replace them with + # empty strings. This is done in a loop to handle multiple raw + # strings on the same line. + while delimiter is None: + # Look for beginning of a raw string. + # See 2.14.15 [lex.string] for syntax. + matched = Match(r'^(.*)\b(?:R|u8R|uR|UR|LR)"([^\s\\()]*)\((.*)$', + line) + if matched: + delimiter = ')' + matched.group(2) + '"' + + end = matched.group(3).find(delimiter) + if end >= 0: + # Raw string ended on same line + line = (matched.group(1) + '""' + + matched.group(3)[end + len(delimiter):]) + delimiter = None + else: + # Start of a multi-line raw string + line = matched.group(1) + '""' + else: + break + + lines_without_raw_strings.append(line) + + # TODO(unknown): if delimiter is not None here, we might want to + # emit a warning for unterminated string. + return lines_without_raw_strings def FindNextMultiLineCommentStart(lines, lineix): - """Find the beginning marker for a multiline comment.""" - while lineix < len(lines): - if lines[lineix].strip().startswith('/*'): - # Only return this marker if the comment goes beyond this line - if lines[lineix].strip().find('*/', 2) < 0: - return lineix - lineix += 1 - return len(lines) + """Find the beginning marker for a multiline comment.""" + while lineix < len(lines): + if lines[lineix].strip().startswith('/*'): + # Only return this marker if the comment goes beyond this line + if lines[lineix].strip().find('*/', 2) < 0: + return lineix + lineix += 1 + return len(lines) def FindNextMultiLineCommentEnd(lines, lineix): - """We are inside a comment, find the end marker.""" - while lineix < len(lines): - if lines[lineix].strip().endswith('*/'): - return lineix - lineix += 1 - return len(lines) + """We are inside a comment, find the end marker.""" + while lineix < len(lines): + if lines[lineix].strip().endswith('*/'): + return lineix + lineix += 1 + return len(lines) def RemoveMultiLineCommentsFromRange(lines, begin, end): - """Clears a range of lines for multi-line comments.""" - # Having // dummy comments makes the lines non-empty, so we will not get - # unnecessary blank line warnings later in the code. - for i in range(begin, end): - lines[i] = '/**/' + """Clears a range of lines for multi-line comments.""" + # Having // dummy comments makes the lines non-empty, so we will not get + # unnecessary blank line warnings later in the code. + for i in range(begin, end): + lines[i] = '/**/' def RemoveMultiLineComments(filename, lines, error): - """Removes multiline (c-style) comments from lines.""" - lineix = 0 - while lineix < len(lines): - lineix_begin = FindNextMultiLineCommentStart(lines, lineix) - if lineix_begin >= len(lines): - return - lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) - if lineix_end >= len(lines): - error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, - 'Could not find end of multi-line comment') - return - RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) - lineix = lineix_end + 1 + """Removes multiline (c-style) comments from lines.""" + lineix = 0 + while lineix < len(lines): + lineix_begin = FindNextMultiLineCommentStart(lines, lineix) + if lineix_begin >= len(lines): + return + lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) + if lineix_end >= len(lines): + error(filename, lineix_begin + 1, 'readability/multiline_comment', + 5, 'Could not find end of multi-line comment') + return + RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) + lineix = lineix_end + 1 def CleanseComments(line): - """Removes //-comments and single-line C-style /* */ comments. + """Removes //-comments and single-line C-style /* */ comments. Args: line: A line of C++ source. @@ -1296,15 +1301,15 @@ def CleanseComments(line): Returns: The line with single-line comments removed. """ - commentpos = line.find('//') - if commentpos != -1 and not IsCppString(line[:commentpos]): - line = line[:commentpos].rstrip() - # get rid of /* ... */ - return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) + commentpos = line.find('//') + if commentpos != -1 and not IsCppString(line[:commentpos]): + line = line[:commentpos].rstrip() + # get rid of /* ... */ + return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) class CleansedLines(object): - """Holds 4 copies of all lines with different preprocessing applied to them. + """Holds 4 copies of all lines with different preprocessing applied to them. 1) elided member contains lines without strings and comments. 2) lines member contains lines without comments. @@ -1314,25 +1319,26 @@ class CleansedLines(object): All these members are of , and of the same length. """ - def __init__(self, lines): - self.elided = [] - self.lines = [] - self.raw_lines = lines - self.num_lines = len(lines) - self.lines_without_raw_strings = CleanseRawStrings(lines) - for linenum in range(len(self.lines_without_raw_strings)): - self.lines.append(CleanseComments( - self.lines_without_raw_strings[linenum])) - elided = self._CollapseStrings(self.lines_without_raw_strings[linenum]) - self.elided.append(CleanseComments(elided)) - - def NumLines(self): - """Returns the number of lines represented.""" - return self.num_lines - - @staticmethod - def _CollapseStrings(elided): - """Collapses strings and chars on a line to simple "" or '' blocks. + def __init__(self, lines): + self.elided = [] + self.lines = [] + self.raw_lines = lines + self.num_lines = len(lines) + self.lines_without_raw_strings = CleanseRawStrings(lines) + for linenum in range(len(self.lines_without_raw_strings)): + self.lines.append( + CleanseComments(self.lines_without_raw_strings[linenum])) + elided = self._CollapseStrings(self.lines_without_raw_strings[ + linenum]) + self.elided.append(CleanseComments(elided)) + + def NumLines(self): + """Returns the number of lines represented.""" + return self.num_lines + + @staticmethod + def _CollapseStrings(elided): + """Collapses strings and chars on a line to simple "" or '' blocks. We nix strings first so we're not fooled by text like '"http://"' @@ -1342,64 +1348,65 @@ class CleansedLines(object): Returns: The line with collapsed strings. """ - if _RE_PATTERN_INCLUDE.match(elided): - return elided - - # Remove escaped characters first to make quote/single quote collapsing - # basic. Things that look like escaped characters shouldn't occur - # outside of strings and chars. - elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) - - # Replace quoted strings and digit separators. Both single quotes - # and double quotes are processed in the same loop, otherwise - # nested quotes wouldn't work. - collapsed = '' - while True: - # Find the first quote character - match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) - if not match: - collapsed += elided - break - head, quote, tail = match.groups() - - if quote == '"': - # Collapse double quoted strings - second_quote = tail.find('"') - if second_quote >= 0: - collapsed += head + '""' - elided = tail[second_quote + 1:] - else: - # Unmatched double quote, don't bother processing the rest - # of the line since this is probably a multiline string. - collapsed += elided - break - else: - # Found single quote, check nearby text to eliminate digit separators. - # - # There is no special handling for floating point here, because - # the integer/fractional/exponent parts would all be parsed - # correctly as long as there are digits on both sides of the - # separator. So we are fine as long as we don't see something - # like "0.'3" (gcc 4.9.0 will not allow this literal). - if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): - match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail) - collapsed += head + match_literal.group(1).replace("'", '') - elided = match_literal.group(2) - else: - second_quote = tail.find('\'') - if second_quote >= 0: - collapsed += head + "''" - elided = tail[second_quote + 1:] - else: - # Unmatched single quote - collapsed += elided - break - - return collapsed + if _RE_PATTERN_INCLUDE.match(elided): + return elided + + # Remove escaped characters first to make quote/single quote collapsing + # basic. Things that look like escaped characters shouldn't occur + # outside of strings and chars. + elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) + + # Replace quoted strings and digit separators. Both single quotes + # and double quotes are processed in the same loop, otherwise + # nested quotes wouldn't work. + collapsed = '' + while True: + # Find the first quote character + match = Match(r'^([^\'"]*)([\'"])(.*)$', elided) + if not match: + collapsed += elided + break + head, quote, tail = match.groups() + + if quote == '"': + # Collapse double quoted strings + second_quote = tail.find('"') + if second_quote >= 0: + collapsed += head + '""' + elided = tail[second_quote + 1:] + else: + # Unmatched double quote, don't bother processing the rest + # of the line since this is probably a multiline string. + collapsed += elided + break + else: + # Found single quote, check nearby text to eliminate digit separators. + # + # There is no special handling for floating point here, because + # the integer/fractional/exponent parts would all be parsed + # correctly as long as there are digits on both sides of the + # separator. So we are fine as long as we don't see something + # like "0.'3" (gcc 4.9.0 will not allow this literal). + if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head): + match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', + "'" + tail) + collapsed += head + match_literal.group(1).replace("'", '') + elided = match_literal.group(2) + else: + second_quote = tail.find('\'') + if second_quote >= 0: + collapsed += head + "''" + elided = tail[second_quote + 1:] + else: + # Unmatched single quote + collapsed += elided + break + + return collapsed def FindEndOfExpressionInLine(line, startpos, stack): - """Find the position just after the end of current parenthesized expression. + """Find the position just after the end of current parenthesized expression. Args: line: a CleansedLines line. @@ -1411,73 +1418,73 @@ def FindEndOfExpressionInLine(line, startpos, stack): On finding an unclosed expression: (-1, None) Otherwise: (-1, new stack at end of this line) """ - for i in xrange(startpos, len(line)): - char = line[i] - if char in '([{': - # Found start of parenthesized expression, push to expression stack - stack.append(char) - elif char == '<': - # Found potential start of template argument list - if i > 0 and line[i - 1] == '<': - # Left shift operator - if stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - elif i > 0 and Search(r'\boperator\s*$', line[0:i]): - # operator<, don't add to stack - continue - else: - # Tentative start of template argument list - stack.append('<') - elif char in ')]}': - # Found end of parenthesized expression. - # - # If we are currently expecting a matching '>', the pending '<' - # must have been an operator. Remove them from expression stack. - while stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - if ((stack[-1] == '(' and char == ')') or - (stack[-1] == '[' and char == ']') or - (stack[-1] == '{' and char == '}')): - stack.pop() - if not stack: - return (i + 1, None) - else: - # Mismatched parentheses - return (-1, None) - elif char == '>': - # Found potential end of template argument list. - - # Ignore "->" and operator functions - if (i > 0 and - (line[i - 1] == '-' or Search(r'\boperator\s*$', line[0:i - 1]))): - continue - - # Pop the stack if there is a matching '<'. Otherwise, ignore - # this '>' since it must be an operator. - if stack: - if stack[-1] == '<': - stack.pop() - if not stack: - return (i + 1, None) - elif char == ';': - # Found something that look like end of statements. If we are currently - # expecting a '>', the matching '<' must have been an operator, since - # template argument list should not contain statements. - while stack and stack[-1] == '<': - stack.pop() - if not stack: - return (-1, None) - - # Did not find end of expression or unbalanced parentheses on this line - return (-1, stack) + for i in xrange(startpos, len(line)): + char = line[i] + if char in '([{': + # Found start of parenthesized expression, push to expression stack + stack.append(char) + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + if stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + elif i > 0 and Search(r'\boperator\s*$', line[0:i]): + # operator<, don't add to stack + continue + else: + # Tentative start of template argument list + stack.append('<') + elif char in ')]}': + # Found end of parenthesized expression. + # + # If we are currently expecting a matching '>', the pending '<' + # must have been an operator. Remove them from expression stack. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + if ((stack[-1] == '(' and char == ')') or + (stack[-1] == '[' and char == ']') or + (stack[-1] == '{' and char == '}')): + stack.pop() + if not stack: + return (i + 1, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == '>': + # Found potential end of template argument list. + + # Ignore "->" and operator functions + if (i > 0 and (line[i - 1] == '-' or Search(r'\boperator\s*$', + line[0:i - 1]))): + continue + + # Pop the stack if there is a matching '<'. Otherwise, ignore + # this '>' since it must be an operator. + if stack: + if stack[-1] == '<': + stack.pop() + if not stack: + return (i + 1, None) + elif char == ';': + # Found something that look like end of statements. If we are currently + # expecting a '>', the matching '<' must have been an operator, since + # template argument list should not contain statements. + while stack and stack[-1] == '<': + stack.pop() + if not stack: + return (-1, None) + + # Did not find end of expression or unbalanced parentheses on this line + return (-1, stack) def CloseExpression(clean_lines, linenum, pos): - """If input points to ( or { or [ or <, finds the position that closes it. + """If input points to ( or { or [ or <, finds the position that closes it. If lines[linenum][pos] points to a '(' or '{' or '[' or '<', finds the linenum/pos that correspond to the closing of the expression. @@ -1499,29 +1506,29 @@ def CloseExpression(clean_lines, linenum, pos): 'cleansed' line at linenum. """ - line = clean_lines.elided[linenum] - if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): - return (line, clean_lines.NumLines(), -1) - - # Check first line - (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) - if end_pos > -1: - return (line, linenum, end_pos) - - # Continue scanning forward - while stack and linenum < clean_lines.NumLines() - 1: - linenum += 1 line = clean_lines.elided[linenum] - (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) + if (line[pos] not in '({[<') or Match(r'<[<=]', line[pos:]): + return (line, clean_lines.NumLines(), -1) + + # Check first line + (end_pos, stack) = FindEndOfExpressionInLine(line, pos, []) if end_pos > -1: - return (line, linenum, end_pos) + return (line, linenum, end_pos) + + # Continue scanning forward + while stack and linenum < clean_lines.NumLines() - 1: + linenum += 1 + line = clean_lines.elided[linenum] + (end_pos, stack) = FindEndOfExpressionInLine(line, 0, stack) + if end_pos > -1: + return (line, linenum, end_pos) - # Did not find end of expression before end of file, give up - return (line, clean_lines.NumLines(), -1) + # Did not find end of expression before end of file, give up + return (line, clean_lines.NumLines(), -1) def FindStartOfExpressionInLine(line, endpos, stack): - """Find position at the matching start of current expression. + """Find position at the matching start of current expression. This is almost the reverse of FindEndOfExpressionInLine, but note that the input position and returned position differs by 1. @@ -1536,69 +1543,68 @@ def FindStartOfExpressionInLine(line, endpos, stack): On finding an unclosed expression: (-1, None) Otherwise: (-1, new stack at beginning of this line) """ - i = endpos - while i >= 0: - char = line[i] - if char in ')]}': - # Found end of expression, push to expression stack - stack.append(char) - elif char == '>': - # Found potential end of template argument list. - # - # Ignore it if it's a "->" or ">=" or "operator>" - if (i > 0 and - (line[i - 1] == '-' or - Match(r'\s>=\s', line[i - 1:]) or - Search(r'\boperator\s*$', line[0:i]))): - i -= 1 - else: - stack.append('>') - elif char == '<': - # Found potential start of template argument list - if i > 0 and line[i - 1] == '<': - # Left shift operator + i = endpos + while i >= 0: + char = line[i] + if char in ')]}': + # Found end of expression, push to expression stack + stack.append(char) + elif char == '>': + # Found potential end of template argument list. + # + # Ignore it if it's a "->" or ">=" or "operator>" + if (i > 0 and + (line[i - 1] == '-' or Match(r'\s>=\s', line[i - 1:]) or + Search(r'\boperator\s*$', line[0:i]))): + i -= 1 + else: + stack.append('>') + elif char == '<': + # Found potential start of template argument list + if i > 0 and line[i - 1] == '<': + # Left shift operator + i -= 1 + else: + # If there is a matching '>', we can pop the expression stack. + # Otherwise, ignore this '<' since it must be an operator. + if stack and stack[-1] == '>': + stack.pop() + if not stack: + return (i, None) + elif char in '([{': + # Found start of expression. + # + # If there are any unmatched '>' on the stack, they must be + # operators. Remove those. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + if ((char == '(' and stack[-1] == ')') or + (char == '[' and stack[-1] == ']') or + (char == '{' and stack[-1] == '}')): + stack.pop() + if not stack: + return (i, None) + else: + # Mismatched parentheses + return (-1, None) + elif char == ';': + # Found something that look like end of statements. If we are currently + # expecting a '<', the matching '>' must have been an operator, since + # template argument list should not contain statements. + while stack and stack[-1] == '>': + stack.pop() + if not stack: + return (-1, None) + i -= 1 - else: - # If there is a matching '>', we can pop the expression stack. - # Otherwise, ignore this '<' since it must be an operator. - if stack and stack[-1] == '>': - stack.pop() - if not stack: - return (i, None) - elif char in '([{': - # Found start of expression. - # - # If there are any unmatched '>' on the stack, they must be - # operators. Remove those. - while stack and stack[-1] == '>': - stack.pop() - if not stack: - return (-1, None) - if ((char == '(' and stack[-1] == ')') or - (char == '[' and stack[-1] == ']') or - (char == '{' and stack[-1] == '}')): - stack.pop() - if not stack: - return (i, None) - else: - # Mismatched parentheses - return (-1, None) - elif char == ';': - # Found something that look like end of statements. If we are currently - # expecting a '<', the matching '>' must have been an operator, since - # template argument list should not contain statements. - while stack and stack[-1] == '>': - stack.pop() - if not stack: - return (-1, None) - - i -= 1 - - return (-1, stack) + + return (-1, stack) def ReverseCloseExpression(clean_lines, linenum, pos): - """If input points to ) or } or ] or >, finds the position that opens it. + """If input points to ) or } or ] or >, finds the position that opens it. If lines[linenum][pos] points to a ')' or '}' or ']' or '>', finds the linenum/pos that correspond to the opening of the expression. @@ -1614,42 +1620,42 @@ def ReverseCloseExpression(clean_lines, linenum, pos): we ignore strings and comments when matching; and the line we return is the 'cleansed' line at linenum. """ - line = clean_lines.elided[linenum] - if line[pos] not in ')}]>': - return (line, 0, -1) - - # Check last line - (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) - if start_pos > -1: - return (line, linenum, start_pos) - - # Continue scanning backward - while stack and linenum > 0: - linenum -= 1 line = clean_lines.elided[linenum] - (start_pos, stack) = FindStartOfExpressionInLine(line, len(line) - 1, stack) - if start_pos > -1: - return (line, linenum, start_pos) + if line[pos] not in ')}]>': + return (line, 0, -1) - # Did not find start of expression before beginning of file, give up - return (line, 0, -1) + # Check last line + (start_pos, stack) = FindStartOfExpressionInLine(line, pos, []) + if start_pos > -1: + return (line, linenum, start_pos) + + # Continue scanning backward + while stack and linenum > 0: + linenum -= 1 + line = clean_lines.elided[linenum] + (start_pos, stack) = FindStartOfExpressionInLine(line, + len(line) - 1, stack) + if start_pos > -1: + return (line, linenum, start_pos) + + # Did not find start of expression before beginning of file, give up + return (line, 0, -1) def CheckForCopyright(filename, lines, error): - """Logs an error if no Copyright message appears at the top of the file.""" + """Logs an error if no Copyright message appears at the top of the file.""" - # We'll say it should occur by line 10. Don't forget there's a - # dummy line at the front. - for line in xrange(1, min(len(lines), 11)): - if re.search(r'Copyright', lines[line], re.I): break - else: # means no copyright line was found - error(filename, 0, 'legal/copyright', 5, - 'No copyright message found. ' - 'You should have a line: "Copyright [year] "') + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. + for line in xrange(1, min(len(lines), 11)): + if re.search(r'Copyright', lines[line], re.I): break + else: # means no copyright line was found + error(filename, 0, 'legal/copyright', 5, 'No copyright message found. ' + 'You should have a line: "Copyright [year] "') def GetIndentLevel(line): - """Return the number of leading spaces in line. + """Return the number of leading spaces in line. Args: line: A string to check. @@ -1657,15 +1663,15 @@ def GetIndentLevel(line): Returns: An integer count of leading spaces, possibly zero. """ - indent = Match(r'^( *)\S', line) - if indent: - return len(indent.group(1)) - else: - return 0 + indent = Match(r'^( *)\S', line) + if indent: + return len(indent.group(1)) + else: + return 0 def GetHeaderGuardCPPVariable(filename): - """Returns the CPP variable that should be used as a header guard. + """Returns the CPP variable that should be used as a header guard. Args: filename: The name of a C++ header file. @@ -1675,12 +1681,12 @@ def GetHeaderGuardCPPVariable(filename): named file. """ - filename = os.path.basename(filename) - return re.sub(r'[^a-zA-Z0-9]', '_', filename).upper() + '_' + filename = os.path.basename(filename) + return re.sub(r'[^a-zA-Z0-9]', '_', filename).upper() + '_' def CheckForHeaderGuard(filename, clean_lines, error): - """Checks that the file contains a header guard. + """Checks that the file contains a header guard. Logs an error if no #ifndef header guard is present. For other headers, checks that the full pathname is used. @@ -1691,123 +1697,124 @@ def CheckForHeaderGuard(filename, clean_lines, error): error: The function to call with any errors found. """ - # Don't check for header guards if there are error suppression - # comments somewhere in this file. - # - # Because this is silencing a warning for a nonexistent line, we - # only support the very specific NOLINT(build/header_guard) syntax, - # and not the general NOLINT or NOLINT(*) syntax. - raw_lines = clean_lines.lines_without_raw_strings - for i in raw_lines: - if Search(r'//\s*NOLINT\(build/header_guard\)', i): - return - - cppvar = GetHeaderGuardCPPVariable(filename) - - ifndef = '' - ifndef_linenum = 0 - define = '' - endif = '' - endif_linenum = 0 - pragma_linenum = -1 - for linenum, line in enumerate(raw_lines): - linesplit = line.split() - if len(linesplit) >= 2: - if linesplit[0] == '#pragma' and linesplit[1] == 'once': - pragma_linenum = linenum - # find the first occurrence of #ifndef and #define, save arg - if not ifndef and linesplit[0] == '#ifndef': - # set ifndef to the header guard presented on the #ifndef line. - ifndef = linesplit[1] - ifndef_linenum = linenum - if not define and linesplit[0] == '#define': - define = linesplit[1] - # find the last occurrence of #endif, save entire line - if line.startswith('#endif'): - endif = line - endif_linenum = linenum - if pragma_linenum != -1: - return # short path for pragma once - if not ifndef or not define or ifndef != define: - error(filename, 0, 'build/header_guard', 5, - 'No #ifndef header guard found, suggested CPP variable is: %s' % - cppvar) - return - - # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ - # for backward compatibility. - if ifndef != cppvar: - error_level = 0 - if ifndef != cppvar + '_': - error_level = 5 - - ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], ifndef_linenum, + # Don't check for header guards if there are error suppression + # comments somewhere in this file. + # + # Because this is silencing a warning for a nonexistent line, we + # only support the very specific NOLINT(build/header_guard) syntax, + # and not the general NOLINT or NOLINT(*) syntax. + raw_lines = clean_lines.lines_without_raw_strings + for i in raw_lines: + if Search(r'//\s*NOLINT\(build/header_guard\)', i): + return + + cppvar = GetHeaderGuardCPPVariable(filename) + + ifndef = '' + ifndef_linenum = 0 + define = '' + endif = '' + endif_linenum = 0 + pragma_linenum = -1 + for linenum, line in enumerate(raw_lines): + linesplit = line.split() + if len(linesplit) >= 2: + if linesplit[0] == '#pragma' and linesplit[1] == 'once': + pragma_linenum = linenum + # find the first occurrence of #ifndef and #define, save arg + if not ifndef and linesplit[0] == '#ifndef': + # set ifndef to the header guard presented on the #ifndef line. + ifndef = linesplit[1] + ifndef_linenum = linenum + if not define and linesplit[0] == '#define': + define = linesplit[1] + # find the last occurrence of #endif, save entire line + if line.startswith('#endif'): + endif = line + endif_linenum = linenum + if pragma_linenum != -1: + return # short path for pragma once + if not ifndef or not define or ifndef != define: + error(filename, 0, 'build/header_guard', 5, + 'No #ifndef header guard found, suggested CPP variable is: %s' % + cppvar) + return + + # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ + # for backward compatibility. + if ifndef != cppvar: + error_level = 0 + if ifndef != cppvar + '_': + error_level = 5 + + ParseNolintSuppressions(filename, raw_lines[ifndef_linenum], + ifndef_linenum, error) + error(filename, ifndef_linenum, 'build/header_guard', error_level, + '#ifndef header guard has wrong style, please use: %s' % cppvar) + + # Check for "//" comments on endif line. + ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, error) - error(filename, ifndef_linenum, 'build/header_guard', error_level, - '#ifndef header guard has wrong style, please use: %s' % cppvar) - - # Check for "//" comments on endif line. - ParseNolintSuppressions(filename, raw_lines[endif_linenum], endif_linenum, - error) - match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) - if match: - if match.group(1) == '_': - # Issue low severity warning for deprecated double trailing underscore - error(filename, endif_linenum, 'build/header_guard', 0, - '#endif line should be "#endif // %s"' % cppvar) - return - - # Didn't find the corresponding "//" comment. If this file does not - # contain any "//" comments at all, it could be that the compiler - # only wants "/**/" comments, look for those instead. - no_single_line_comments = True - for i in xrange(1, len(raw_lines) - 1): - line = raw_lines[i] - if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', line): - no_single_line_comments = False - break - - if no_single_line_comments: - match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) + match = Match(r'#endif\s*//\s*' + cppvar + r'(_)?\b', endif) if match: - if match.group(1) == '_': - # Low severity warning for double trailing underscore - error(filename, endif_linenum, 'build/header_guard', 0, - '#endif line should be "#endif /* %s */"' % cppvar) - return + if match.group(1) == '_': + # Issue low severity warning for deprecated double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif // %s"' % cppvar) + return - # Didn't find anything - error(filename, endif_linenum, 'build/header_guard', 5, - '#endif line should be "#endif // %s"' % cppvar) + # Didn't find the corresponding "//" comment. If this file does not + # contain any "//" comments at all, it could be that the compiler + # only wants "/**/" comments, look for those instead. + no_single_line_comments = True + for i in xrange(1, len(raw_lines) - 1): + line = raw_lines[i] + if Match(r'^(?:(?:\'(?:\.|[^\'])*\')|(?:"(?:\.|[^"])*")|[^\'"])*//', + line): + no_single_line_comments = False + break + + if no_single_line_comments: + match = Match(r'#endif\s*/\*\s*' + cppvar + r'(_)?\s*\*/', endif) + if match: + if match.group(1) == '_': + # Low severity warning for double trailing underscore + error(filename, endif_linenum, 'build/header_guard', 0, + '#endif line should be "#endif /* %s */"' % cppvar) + return + + # Didn't find anything + error(filename, endif_linenum, 'build/header_guard', 5, + '#endif line should be "#endif // %s"' % cppvar) def CheckHeaderFileIncluded(filename, include_state, error): - """Logs an error if a .cc file does not include its header.""" - - # Do not check test files - if filename.endswith('_test.cc') or filename.endswith('_unittest.cc'): - return - - fileinfo = FileInfo(filename) - headerfile = filename[0:len(filename) - 2] + 'h' - if not os.path.exists(headerfile): - return - headername = FileInfo(headerfile).RepositoryName() - first_include = 0 - for section_list in include_state.include_list: - for f in section_list: - if headername in f[0] or f[0] in headername: + """Logs an error if a .cc file does not include its header.""" + + # Do not check test files + if filename.endswith('_test.cc') or filename.endswith('_unittest.cc'): + return + + fileinfo = FileInfo(filename) + headerfile = filename[0:len(filename) - 2] + 'h' + if not os.path.exists(headerfile): return - if not first_include: - first_include = f[1] + headername = FileInfo(headerfile).RepositoryName() + first_include = 0 + for section_list in include_state.include_list: + for f in section_list: + if headername in f[0] or f[0] in headername: + return + if not first_include: + first_include = f[1] - error(filename, first_include, 'build/include', 5, - '%s should include its header file %s' % (fileinfo.RepositoryName(), - headername)) + error(filename, first_include, 'build/include', 5, + '%s should include its header file %s' % (fileinfo.RepositoryName(), + headername)) def CheckForBadCharacters(filename, lines, error): - """Logs an error for each line containing bad characters. + """Logs an error for each line containing bad characters. Two kinds of bad characters: @@ -1823,16 +1830,19 @@ def CheckForBadCharacters(filename, lines, error): lines: An array of strings, each representing a line of the file. error: The function to call with any errors found. """ - for linenum, line in enumerate(lines): - if u'\ufffd' in line: - error(filename, linenum, 'readability/utf8', 5, - 'Line contains invalid UTF-8 (or Unicode replacement character).') - if '\0' in line: - error(filename, linenum, 'readability/nul', 5, 'Line contains NUL byte.') + for linenum, line in enumerate(lines): + if u'\ufffd' in line: + error( + filename, linenum, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).' + ) + if '\0' in line: + error(filename, linenum, 'readability/nul', 5, + 'Line contains NUL byte.') def CheckForNewlineAtEOF(filename, lines, error): - """Logs an error if there is no newline char at the end of the file. + """Logs an error if there is no newline char at the end of the file. Args: filename: The name of the current file. @@ -1840,17 +1850,18 @@ def CheckForNewlineAtEOF(filename, lines, error): error: The function to call with any errors found. """ - # The array lines() was created by adding two newlines to the - # original file (go figure), then splitting on \n. - # To verify that the file ends in \n, we just have to make sure the - # last-but-two element of lines() exists and is empty. - if len(lines) < 3 or lines[-2]: - error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, - 'Could not find a newline character at the end of the file.') + # The array lines() was created by adding two newlines to the + # original file (go figure), then splitting on \n. + # To verify that the file ends in \n, we just have to make sure the + # last-but-two element of lines() exists and is empty. + if len(lines) < 3 or lines[-2]: + error(filename, + len(lines) - 2, 'whitespace/ending_newline', 5, + 'Could not find a newline character at the end of the file.') def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): - """Logs an error if we see /* ... */ or "..." that extend past one line. + """Logs an error if we see /* ... */ or "..." that extend past one line. /* ... */ comments are legit inside macros, for one line. Otherwise, we prefer // comments, so it's ok to warn about the @@ -1866,25 +1877,25 @@ def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Remove all \\ (escaped backslashes) from the line. They are OK, and the - # second (escaped) slash may trigger later \" detection erroneously. - line = line.replace('\\\\', '') + # Remove all \\ (escaped backslashes) from the line. They are OK, and the + # second (escaped) slash may trigger later \" detection erroneously. + line = line.replace('\\\\', '') - if line.count('/*') > line.count('*/'): - error(filename, linenum, 'readability/multiline_comment', 5, - 'Complex multi-line /*...*/-style comment found. ' - 'Lint may give bogus warnings. ' - 'Consider replacing these with //-style comments, ' - 'with #if 0...#endif, ' - 'or with more clearly structured multi-line comments.') + if line.count('/*') > line.count('*/'): + error(filename, linenum, 'readability/multiline_comment', 5, + 'Complex multi-line /*...*/-style comment found. ' + 'Lint may give bogus warnings. ' + 'Consider replacing these with //-style comments, ' + 'with #if 0...#endif, ' + 'or with more clearly structured multi-line comments.') - if (line.count('"') - line.count('\\"')) % 2: - error(filename, linenum, 'readability/multiline_string', 5, - 'Multi-line string ("...") found. This lint script doesn\'t ' - 'do well with such strings, and may give bogus warnings. ' - 'Use C++11 raw strings or concatenation instead.') + if (line.count('"') - line.count('\\"')) % 2: + error(filename, linenum, 'readability/multiline_string', 5, + 'Multi-line string ("...") found. This lint script doesn\'t ' + 'do well with such strings, and may give bogus warnings. ' + 'Use C++11 raw strings or concatenation instead.') # (non-threadsafe name, thread-safe alternative, validation pattern) @@ -1911,14 +1922,12 @@ _THREADING_LIST = ( ('gmtime(', 'gmtime_r(', _UNSAFE_FUNC_PREFIX + r'gmtime\([^)]+\)'), ('localtime(', 'localtime_r(', _UNSAFE_FUNC_PREFIX + r'localtime\([^)]+\)'), ('rand(', 'rand_r(', _UNSAFE_FUNC_PREFIX + r'rand\(\)'), - ('strtok(', 'strtok_r(', - _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), - ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), - ) + ('strtok(', 'strtok_r(', _UNSAFE_FUNC_PREFIX + r'strtok\([^)]+\)'), + ('ttyname(', 'ttyname_r(', _UNSAFE_FUNC_PREFIX + r'ttyname\([^)]+\)'), ) def CheckPosixThreading(filename, clean_lines, linenum, error): - """Checks for calls to thread-unsafe functions. + """Checks for calls to thread-unsafe functions. Much code has been originally written without consideration of multi-threading. Also, engineers are relying on their old experience; @@ -1932,19 +1941,18 @@ def CheckPosixThreading(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: - # Additional pattern matching check to confirm that this is the - # function we are looking for - if Search(pattern, line): - error(filename, linenum, 'runtime/threadsafe_fn', 2, - 'Consider using ' + multithread_safe_func + - '...) instead of ' + single_thread_func + - '...) for improved thread safety.') + line = clean_lines.elided[linenum] + for single_thread_func, multithread_safe_func, pattern in _THREADING_LIST: + # Additional pattern matching check to confirm that this is the + # function we are looking for + if Search(pattern, line): + error(filename, linenum, 'runtime/threadsafe_fn', 2, + 'Consider using ' + multithread_safe_func + '...) instead of ' + + single_thread_func + '...) for improved thread safety.') def CheckVlogArguments(filename, clean_lines, linenum, error): - """Checks that VLOG() is only used for defining a logging level. + """Checks that VLOG() is only used for defining a logging level. For example, VLOG(2) is correct. VLOG(INFO), VLOG(WARNING), VLOG(ERROR), and VLOG(FATAL) are not. @@ -1955,20 +1963,20 @@ def CheckVlogArguments(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): - error(filename, linenum, 'runtime/vlog', 5, - 'VLOG() should be used with numeric verbosity level. ' - 'Use LOG() if you want symbolic severity levels.') + line = clean_lines.elided[linenum] + if Search(r'\bVLOG\((INFO|ERROR|WARNING|DFATAL|FATAL)\)', line): + error(filename, linenum, 'runtime/vlog', 5, + 'VLOG() should be used with numeric verbosity level. ' + 'Use LOG() if you want symbolic severity levels.') + # Matches invalid increment: *count++, which moves pointer instead of # incrementing a value. -_RE_PATTERN_INVALID_INCREMENT = re.compile( - r'^\s*\*\w+(\+\+|--);') +_RE_PATTERN_INVALID_INCREMENT = re.compile(r'^\s*\*\w+(\+\+|--);') def CheckInvalidIncrement(filename, clean_lines, linenum, error): - """Checks for invalid increment *count++. + """Checks for invalid increment *count++. For example following function: void increment_counter(int* count) { @@ -1983,37 +1991,38 @@ def CheckInvalidIncrement(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - if _RE_PATTERN_INVALID_INCREMENT.match(line): - error(filename, linenum, 'runtime/invalid_increment', 5, - 'Changing pointer instead of value (or unused value of operator*).') + line = clean_lines.elided[linenum] + if _RE_PATTERN_INVALID_INCREMENT.match(line): + error( + filename, linenum, 'runtime/invalid_increment', 5, + 'Changing pointer instead of value (or unused value of operator*).') def IsMacroDefinition(clean_lines, linenum): - if Search(r'^#define', clean_lines[linenum]): - return True + if Search(r'^#define', clean_lines[linenum]): + return True - if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): - return True + if linenum > 0 and Search(r'\\$', clean_lines[linenum - 1]): + return True - return False + return False def IsForwardClassDeclaration(clean_lines, linenum): - return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) + return Match(r'^\s*(\btemplate\b)*.*class\s+\w+;\s*$', clean_lines[linenum]) class _BlockInfo(object): - """Stores information about a generic block of code.""" + """Stores information about a generic block of code.""" - def __init__(self, seen_open_brace): - self.seen_open_brace = seen_open_brace - self.open_parentheses = 0 - self.inline_asm = _NO_ASM - self.check_namespace_indentation = False + def __init__(self, seen_open_brace): + self.seen_open_brace = seen_open_brace + self.open_parentheses = 0 + self.inline_asm = _NO_ASM + self.check_namespace_indentation = False - def CheckBegin(self, filename, clean_lines, linenum, error): - """Run checks that applies to text up to the opening brace. + def CheckBegin(self, filename, clean_lines, linenum, error): + """Run checks that applies to text up to the opening brace. This is mostly for checking the text after the class identifier and the "{", usually where the base class is specified. For other @@ -2025,10 +2034,10 @@ class _BlockInfo(object): linenum: The number of the line to check. error: The function to call with any errors found. """ - pass + pass - def CheckEnd(self, filename, clean_lines, linenum, error): - """Run checks that applies to text after the closing brace. + def CheckEnd(self, filename, clean_lines, linenum, error): + """Run checks that applies to text after the closing brace. This is mostly used for checking end of namespace comments. @@ -2038,10 +2047,10 @@ class _BlockInfo(object): linenum: The number of the line to check. error: The function to call with any errors found. """ - pass + pass - def IsBlockInfo(self): - """Returns true if this block is a _BlockInfo. + def IsBlockInfo(self): + """Returns true if this block is a _BlockInfo. This is convenient for verifying that an object is an instance of a _BlockInfo, but not an instance of any of the derived classes. @@ -2049,231 +2058,235 @@ class _BlockInfo(object): Returns: True for this class, False for derived classes. """ - return self.__class__ == _BlockInfo + return self.__class__ == _BlockInfo class _ExternCInfo(_BlockInfo): - """Stores information about an 'extern "C"' block.""" + """Stores information about an 'extern "C"' block.""" - def __init__(self): - _BlockInfo.__init__(self, True) + def __init__(self): + _BlockInfo.__init__(self, True) class _ClassInfo(_BlockInfo): - """Stores information about a class.""" - - def __init__(self, name, class_or_struct, clean_lines, linenum): - _BlockInfo.__init__(self, False) - self.name = name - self.starting_linenum = linenum - self.is_derived = False - self.check_namespace_indentation = True - if class_or_struct == 'struct': - self.access = 'public' - self.is_struct = True - else: - self.access = 'private' - self.is_struct = False + """Stores information about a class.""" + + def __init__(self, name, class_or_struct, clean_lines, linenum): + _BlockInfo.__init__(self, False) + self.name = name + self.starting_linenum = linenum + self.is_derived = False + self.check_namespace_indentation = True + if class_or_struct == 'struct': + self.access = 'public' + self.is_struct = True + else: + self.access = 'private' + self.is_struct = False - # Remember initial indentation level for this class. Using raw_lines here - # instead of elided to account for leading comments. - self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) + # Remember initial indentation level for this class. Using raw_lines here + # instead of elided to account for leading comments. + self.class_indent = GetIndentLevel(clean_lines.raw_lines[linenum]) - # Try to find the end of the class. This will be confused by things like: - # class A { - # } *x = { ... - # - # But it's still good enough for CheckSectionSpacing. - self.last_line = 0 - depth = 0 - for i in range(linenum, clean_lines.NumLines()): - line = clean_lines.elided[i] - depth += line.count('{') - line.count('}') - if not depth: - self.last_line = i - break - - def CheckBegin(self, filename, clean_lines, linenum, error): - # Look for a bare ':' - if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): - self.is_derived = True - - def CheckEnd(self, filename, clean_lines, linenum, error): - # If there is a DISALLOW macro, it should appear near the end of - # the class. - seen_last_thing_in_class = False - for i in xrange(linenum - 1, self.starting_linenum, -1): - match = Search( - r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + - self.name + r'\)', - clean_lines.elided[i]) - if match: - if seen_last_thing_in_class: - error(filename, i, 'readability/constructors', 3, - match.group(1) + ' should be the last thing in the class') - break - - if not Match(r'^\s*$', clean_lines.elided[i]): - seen_last_thing_in_class = True - - # Check that closing brace is aligned with beginning of the class. - # Only do this if the closing brace is indented by only whitespaces. - # This means we will not check single-line class definitions. - indent = Match(r'^( *)\}', clean_lines.elided[linenum]) - if indent and len(indent.group(1)) != self.class_indent: - if self.is_struct: - parent = 'struct ' + self.name - else: - parent = 'class ' + self.name - error(filename, linenum, 'whitespace/indent', 3, - 'Closing brace should be aligned with beginning of %s' % parent) + # Try to find the end of the class. This will be confused by things like: + # class A { + # } *x = { ... + # + # But it's still good enough for CheckSectionSpacing. + self.last_line = 0 + depth = 0 + for i in range(linenum, clean_lines.NumLines()): + line = clean_lines.elided[i] + depth += line.count('{') - line.count('}') + if not depth: + self.last_line = i + break + + def CheckBegin(self, filename, clean_lines, linenum, error): + # Look for a bare ':' + if Search('(^|[^:]):($|[^:])', clean_lines.elided[linenum]): + self.is_derived = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + # If there is a DISALLOW macro, it should appear near the end of + # the class. + seen_last_thing_in_class = False + for i in xrange(linenum - 1, self.starting_linenum, -1): + match = Search( + r'\b(DISALLOW_COPY_AND_ASSIGN|DISALLOW_IMPLICIT_CONSTRUCTORS)\(' + + self.name + r'\)', clean_lines.elided[i]) + if match: + if seen_last_thing_in_class: + error(filename, i, 'readability/constructors', 3, + match.group(1) + + ' should be the last thing in the class') + break + + if not Match(r'^\s*$', clean_lines.elided[i]): + seen_last_thing_in_class = True + + # Check that closing brace is aligned with beginning of the class. + # Only do this if the closing brace is indented by only whitespaces. + # This means we will not check single-line class definitions. + indent = Match(r'^( *)\}', clean_lines.elided[linenum]) + if indent and len(indent.group(1)) != self.class_indent: + if self.is_struct: + parent = 'struct ' + self.name + else: + parent = 'class ' + self.name + error(filename, linenum, 'whitespace/indent', 3, + 'Closing brace should be aligned with beginning of %s' % + parent) class _NamespaceInfo(_BlockInfo): - """Stores information about a namespace.""" - - def __init__(self, name, linenum): - _BlockInfo.__init__(self, False) - self.name = name or '' - self.starting_linenum = linenum - self.check_namespace_indentation = True - - def CheckEnd(self, filename, clean_lines, linenum, error): - """Check end of namespace comments.""" - line = clean_lines.raw_lines[linenum] - - # Check how many lines is enclosed in this namespace. Don't issue - # warning for missing namespace comments if there aren't enough - # lines. However, do apply checks if there is already an end of - # namespace comment and it's incorrect. - # - # TODO(unknown): We always want to check end of namespace comments - # if a namespace is large, but sometimes we also want to apply the - # check if a short namespace contained nontrivial things (something - # other than forward declarations). There is currently no logic on - # deciding what these nontrivial things are, so this check is - # triggered by namespace size only, which works most of the time. - if (linenum - self.starting_linenum < 10 - and not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)): - return - - # Look for matching comment at end of namespace. - # - # Note that we accept C style "/* */" comments for terminating - # namespaces, so that code that terminate namespaces inside - # preprocessor macros can be cpplint clean. - # - # We also accept stuff like "// end of namespace ." with the - # period at the end. - # - # Besides these, we don't accept anything else, otherwise we might - # get false negatives when existing comment is a substring of the - # expected namespace. - if self.name: - # Named namespace - if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + re.escape(self.name) + - r'[\*/\.\\\s]*$'), - line): - error(filename, linenum, 'readability/namespace', 5, - 'Namespace should be terminated with "// namespace %s"' % - self.name) - else: - # Anonymous namespace - if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): - # If "// namespace anonymous" or "// anonymous namespace (more text)", - # mention "// anonymous namespace" as an acceptable form - if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', line): - error(filename, linenum, 'readability/namespace', 5, - 'Anonymous namespace should be terminated with "// namespace"' - ' or "// anonymous namespace"') + """Stores information about a namespace.""" + + def __init__(self, name, linenum): + _BlockInfo.__init__(self, False) + self.name = name or '' + self.starting_linenum = linenum + self.check_namespace_indentation = True + + def CheckEnd(self, filename, clean_lines, linenum, error): + """Check end of namespace comments.""" + line = clean_lines.raw_lines[linenum] + + # Check how many lines is enclosed in this namespace. Don't issue + # warning for missing namespace comments if there aren't enough + # lines. However, do apply checks if there is already an end of + # namespace comment and it's incorrect. + # + # TODO(unknown): We always want to check end of namespace comments + # if a namespace is large, but sometimes we also want to apply the + # check if a short namespace contained nontrivial things (something + # other than forward declarations). There is currently no logic on + # deciding what these nontrivial things are, so this check is + # triggered by namespace size only, which works most of the time. + if (linenum - self.starting_linenum < 10 and + not Match(r'};*\s*(//|/\*).*\bnamespace\b', line)): + return + + # Look for matching comment at end of namespace. + # + # Note that we accept C style "/* */" comments for terminating + # namespaces, so that code that terminate namespaces inside + # preprocessor macros can be cpplint clean. + # + # We also accept stuff like "// end of namespace ." with the + # period at the end. + # + # Besides these, we don't accept anything else, otherwise we might + # get false negatives when existing comment is a substring of the + # expected namespace. + if self.name: + # Named namespace + if not Match((r'};*\s*(//|/\*).*\bnamespace\s+' + + re.escape(self.name) + r'[\*/\.\\\s]*$'), line): + error(filename, linenum, 'readability/namespace', 5, + 'Namespace should be terminated with "// namespace %s"' % + self.name) else: - error(filename, linenum, 'readability/namespace', 5, - 'Anonymous namespace should be terminated with "// namespace"') + # Anonymous namespace + if not Match(r'};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line): + # If "// namespace anonymous" or "// anonymous namespace (more text)", + # mention "// anonymous namespace" as an acceptable form + if Match(r'}.*\b(namespace anonymous|anonymous namespace)\b', + line): + error( + filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"' + ' or "// anonymous namespace"') + else: + error( + filename, linenum, 'readability/namespace', 5, + 'Anonymous namespace should be terminated with "// namespace"' + ) class _PreprocessorInfo(object): - """Stores checkpoints of nesting stacks when #if/#else is seen.""" + """Stores checkpoints of nesting stacks when #if/#else is seen.""" - def __init__(self, stack_before_if): - # The entire nesting stack before #if - self.stack_before_if = stack_before_if + def __init__(self, stack_before_if): + # The entire nesting stack before #if + self.stack_before_if = stack_before_if - # The entire nesting stack up to #else - self.stack_before_else = [] + # The entire nesting stack up to #else + self.stack_before_else = [] - # Whether we have already seen #else or #elif - self.seen_else = False + # Whether we have already seen #else or #elif + self.seen_else = False class NestingState(object): - """Holds states related to parsing braces.""" - - def __init__(self): - # Stack for tracking all braces. An object is pushed whenever we - # see a "{", and popped when we see a "}". Only 3 types of - # objects are possible: - # - _ClassInfo: a class or struct. - # - _NamespaceInfo: a namespace. - # - _BlockInfo: some other type of block. - self.stack = [] - - # Top of the previous stack before each Update(). - # - # Because the nesting_stack is updated at the end of each line, we - # had to do some convoluted checks to find out what is the current - # scope at the beginning of the line. This check is simplified by - # saving the previous top of nesting stack. - # - # We could save the full stack, but we only need the top. Copying - # the full nesting stack would slow down cpplint by ~10%. - self.previous_stack_top = [] + """Holds states related to parsing braces.""" + + def __init__(self): + # Stack for tracking all braces. An object is pushed whenever we + # see a "{", and popped when we see a "}". Only 3 types of + # objects are possible: + # - _ClassInfo: a class or struct. + # - _NamespaceInfo: a namespace. + # - _BlockInfo: some other type of block. + self.stack = [] + + # Top of the previous stack before each Update(). + # + # Because the nesting_stack is updated at the end of each line, we + # had to do some convoluted checks to find out what is the current + # scope at the beginning of the line. This check is simplified by + # saving the previous top of nesting stack. + # + # We could save the full stack, but we only need the top. Copying + # the full nesting stack would slow down cpplint by ~10%. + self.previous_stack_top = [] - # Stack of _PreprocessorInfo objects. - self.pp_stack = [] + # Stack of _PreprocessorInfo objects. + self.pp_stack = [] - def SeenOpenBrace(self): - """Check if we have seen the opening brace for the innermost block. + def SeenOpenBrace(self): + """Check if we have seen the opening brace for the innermost block. Returns: True if we have seen the opening brace, False if the innermost block is still expecting an opening brace. """ - return (not self.stack) or self.stack[-1].seen_open_brace + return (not self.stack) or self.stack[-1].seen_open_brace - def InNamespaceBody(self): - """Check if we are currently one level inside a namespace body. + def InNamespaceBody(self): + """Check if we are currently one level inside a namespace body. Returns: True if top of the stack is a namespace block, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _NamespaceInfo) + return self.stack and isinstance(self.stack[-1], _NamespaceInfo) - def InExternC(self): - """Check if we are currently one level inside an 'extern "C"' block. + def InExternC(self): + """Check if we are currently one level inside an 'extern "C"' block. Returns: True if top of the stack is an extern block, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _ExternCInfo) + return self.stack and isinstance(self.stack[-1], _ExternCInfo) - def InClassDeclaration(self): - """Check if we are currently one level inside a class or struct declaration. + def InClassDeclaration(self): + """Check if we are currently one level inside a class or struct declaration. Returns: True if top of the stack is a class/struct, False otherwise. """ - return self.stack and isinstance(self.stack[-1], _ClassInfo) + return self.stack and isinstance(self.stack[-1], _ClassInfo) - def InAsmBlock(self): - """Check if we are currently one level inside an inline ASM block. + def InAsmBlock(self): + """Check if we are currently one level inside an inline ASM block. Returns: True if the top of the stack is a block containing inline ASM. """ - return self.stack and self.stack[-1].inline_asm != _NO_ASM + return self.stack and self.stack[-1].inline_asm != _NO_ASM - def InTemplateArgumentList(self, clean_lines, linenum, pos): - """Check if current position is inside template argument list. + def InTemplateArgumentList(self, clean_lines, linenum, pos): + """Check if current position is inside template argument list. Args: clean_lines: A CleansedLines instance containing the file. @@ -2282,50 +2295,51 @@ class NestingState(object): Returns: True if (linenum, pos) is inside template arguments. """ - while linenum < clean_lines.NumLines(): - # Find the earliest character that might indicate a template argument - line = clean_lines.elided[linenum] - match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) - if not match: - linenum += 1 - pos = 0 - continue - token = match.group(1) - pos += len(match.group(0)) - - # These things do not look like template argument list: - # class Suspect { - # class Suspect x; } - if token in ('{', '}', ';'): return False - - # These things look like template argument list: - # template - # template - # template - # template - if token in ('>', '=', '[', ']', '.'): return True - - # Check if token is an unmatched '<'. - # If not, move on to the next character. - if token != '<': - pos += 1 - if pos >= len(line): - linenum += 1 - pos = 0 - continue - - # We can't be sure if we just find a single '<', and need to - # find the matching '>'. - (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1) - if end_pos < 0: - # Not sure if template argument list or syntax error in file + while linenum < clean_lines.NumLines(): + # Find the earliest character that might indicate a template argument + line = clean_lines.elided[linenum] + match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:]) + if not match: + linenum += 1 + pos = 0 + continue + token = match.group(1) + pos += len(match.group(0)) + + # These things do not look like template argument list: + # class Suspect { + # class Suspect x; } + if token in ('{', '}', ';'): return False + + # These things look like template argument list: + # template + # template + # template + # template + if token in ('>', '=', '[', ']', '.'): return True + + # Check if token is an unmatched '<'. + # If not, move on to the next character. + if token != '<': + pos += 1 + if pos >= len(line): + linenum += 1 + pos = 0 + continue + + # We can't be sure if we just find a single '<', and need to + # find the matching '>'. + (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, + pos - 1) + if end_pos < 0: + # Not sure if template argument list or syntax error in file + return False + linenum = end_line + pos = end_pos return False - linenum = end_line - pos = end_pos - return False - def UpdatePreprocessor(self, line): - """Update preprocessor stack. + def UpdatePreprocessor(self, line): + """Update preprocessor stack. We need to handle preprocessors due to classes like this: #ifdef SWIG @@ -2345,44 +2359,45 @@ class NestingState(object): Args: line: current line to check. """ - if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): - # Beginning of #if block, save the nesting stack here. The saved - # stack will allow us to restore the parsing state in the #else case. - self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) - elif Match(r'^\s*#\s*(else|elif)\b', line): - # Beginning of #else block - if self.pp_stack: - if not self.pp_stack[-1].seen_else: - # This is the first #else or #elif block. Remember the - # whole nesting stack up to this point. This is what we - # keep after the #endif. - self.pp_stack[-1].seen_else = True - self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack) - - # Restore the stack to how it was before the #if - self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) - else: - # TODO(unknown): unexpected #else, issue warning? - pass - elif Match(r'^\s*#\s*endif\b', line): - # End of #if or #else blocks. - if self.pp_stack: - # If we saw an #else, we will need to restore the nesting - # stack to its former state before the #else, otherwise we - # will just continue from where we left off. - if self.pp_stack[-1].seen_else: - # Here we can just use a shallow copy since we are the last - # reference to it. - self.stack = self.pp_stack[-1].stack_before_else - # Drop the corresponding #if - self.pp_stack.pop() - else: - # TODO(unknown): unexpected #endif, issue warning? - pass - - # TODO(unknown): Update() is too long, but we will refactor later. - def Update(self, filename, clean_lines, linenum, error): - """Update nesting state with current line. + if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line): + # Beginning of #if block, save the nesting stack here. The saved + # stack will allow us to restore the parsing state in the #else case. + self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack))) + elif Match(r'^\s*#\s*(else|elif)\b', line): + # Beginning of #else block + if self.pp_stack: + if not self.pp_stack[-1].seen_else: + # This is the first #else or #elif block. Remember the + # whole nesting stack up to this point. This is what we + # keep after the #endif. + self.pp_stack[-1].seen_else = True + self.pp_stack[-1].stack_before_else = copy.deepcopy( + self.stack) + + # Restore the stack to how it was before the #if + self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if) + else: + # TODO(unknown): unexpected #else, issue warning? + pass + elif Match(r'^\s*#\s*endif\b', line): + # End of #if or #else blocks. + if self.pp_stack: + # If we saw an #else, we will need to restore the nesting + # stack to its former state before the #else, otherwise we + # will just continue from where we left off. + if self.pp_stack[-1].seen_else: + # Here we can just use a shallow copy since we are the last + # reference to it. + self.stack = self.pp_stack[-1].stack_before_else + # Drop the corresponding #if + self.pp_stack.pop() + else: + # TODO(unknown): unexpected #endif, issue warning? + pass + + # TODO(unknown): Update() is too long, but we will refactor later. + def Update(self, filename, clean_lines, linenum, error): + """Update nesting state with current line. Args: filename: The name of the current file. @@ -2390,198 +2405,201 @@ class NestingState(object): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Remember top of the previous nesting stack. - # - # The stack is always pushed/popped and not modified in place, so - # we can just do a shallow copy instead of copy.deepcopy. Using - # deepcopy would slow down cpplint by ~28%. - if self.stack: - self.previous_stack_top = self.stack[-1] - else: - self.previous_stack_top = None - - # Update pp_stack - self.UpdatePreprocessor(line) - - # Count parentheses. This is to avoid adding struct arguments to - # the nesting stack. - if self.stack: - inner_block = self.stack[-1] - depth_change = line.count('(') - line.count(')') - inner_block.open_parentheses += depth_change - - # Also check if we are starting or ending an inline assembly block. - if inner_block.inline_asm in (_NO_ASM, _END_ASM): - if (depth_change != 0 and - inner_block.open_parentheses == 1 and - _MATCH_ASM.match(line)): - # Enter assembly block - inner_block.inline_asm = _INSIDE_ASM - else: - # Not entering assembly block. If previous line was _END_ASM, - # we will now shift to _NO_ASM state. - inner_block.inline_asm = _NO_ASM - elif (inner_block.inline_asm == _INSIDE_ASM and - inner_block.open_parentheses == 0): - # Exit assembly block - inner_block.inline_asm = _END_ASM - - # Consume namespace declaration at the beginning of the line. Do - # this in a loop so that we catch same line declarations like this: - # namespace proto2 { namespace bridge { class MessageSet; } } - while True: - # Match start of namespace. The "\b\s*" below catches namespace - # declarations even if it weren't followed by a whitespace, this - # is so that we don't confuse our namespace checker. The - # missing spaces will be flagged by CheckSpacing. - namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', line) - if not namespace_decl_match: - break - - new_namespace = _NamespaceInfo(namespace_decl_match.group(1), linenum) - self.stack.append(new_namespace) - - line = namespace_decl_match.group(2) - if line.find('{') != -1: - new_namespace.seen_open_brace = True - line = line[line.find('{') + 1:] - - # Look for a class declaration in whatever is left of the line - # after parsing namespaces. The regexp accounts for decorated classes - # such as in: - # class LOCKABLE API Object { - # }; - class_decl_match = Match( - r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?' - r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))' - r'(.*)$', line) - if (class_decl_match and - (not self.stack or self.stack[-1].open_parentheses == 0)): - # We do not want to accept classes that are actually template arguments: - # template , - # template class Ignore3> - # void Function() {}; - # - # To avoid template argument cases, we scan forward and look for - # an unmatched '>'. If we see one, assume we are inside a - # template argument list. - end_declaration = len(class_decl_match.group(1)) - if not self.InTemplateArgumentList(clean_lines, linenum, end_declaration): - self.stack.append(_ClassInfo( - class_decl_match.group(3), class_decl_match.group(2), - clean_lines, linenum)) - line = class_decl_match.group(4) - - # If we have not yet seen the opening brace for the innermost block, - # run checks here. - if not self.SeenOpenBrace(): - self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) - - # Update access control if we are inside a class/struct - if self.stack and isinstance(self.stack[-1], _ClassInfo): - classinfo = self.stack[-1] - access_match = Match( - r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' - r':(?:[^:]|$)', - line) - if access_match: - classinfo.access = access_match.group(2) - - # Check that access keywords are indented +1 space. Skip this - # check if the keywords are not preceded by whitespaces. - indent = access_match.group(1) - if (len(indent) != classinfo.class_indent + 1 and - Match(r'^\s*$', indent)): - if classinfo.is_struct: - parent = 'struct ' + classinfo.name - else: - parent = 'class ' + classinfo.name - slots = '' - if access_match.group(3): - slots = access_match.group(3) - error(filename, linenum, 'whitespace/indent', 3, - '%s%s: should be indented +1 space inside %s' % ( - access_match.group(2), slots, parent)) - - # Consume braces or semicolons from what's left of the line - while True: - # Match first brace, semicolon, or closed parenthesis. - matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) - if not matched: - break - - token = matched.group(1) - if token == '{': - # If namespace or class hasn't seen a opening brace yet, mark - # namespace/class head as complete. Push a new block onto the - # stack otherwise. - if not self.SeenOpenBrace(): - self.stack[-1].seen_open_brace = True - elif Match(r'^extern\s*"[^"]*"\s*\{', line): - self.stack.append(_ExternCInfo()) - else: - self.stack.append(_BlockInfo(True)) - if _MATCH_ASM.match(line): - self.stack[-1].inline_asm = _BLOCK_ASM - - elif token == ';' or token == ')': - # If we haven't seen an opening brace yet, but we already saw - # a semicolon, this is probably a forward declaration. Pop - # the stack for these. + # Remember top of the previous nesting stack. # - # Similarly, if we haven't seen an opening brace yet, but we - # already saw a closing parenthesis, then these are probably - # function arguments with extra "class" or "struct" keywords. - # Also pop these stack for these. - if not self.SeenOpenBrace(): - self.stack.pop() - else: # token == '}' - # Perform end of block checks and pop the stack. + # The stack is always pushed/popped and not modified in place, so + # we can just do a shallow copy instead of copy.deepcopy. Using + # deepcopy would slow down cpplint by ~28%. if self.stack: - self.stack[-1].CheckEnd(filename, clean_lines, linenum, error) - self.stack.pop() - line = matched.group(2) + self.previous_stack_top = self.stack[-1] + else: + self.previous_stack_top = None + + # Update pp_stack + self.UpdatePreprocessor(line) - def InnermostClass(self): - """Get class info on the top of the stack. + # Count parentheses. This is to avoid adding struct arguments to + # the nesting stack. + if self.stack: + inner_block = self.stack[-1] + depth_change = line.count('(') - line.count(')') + inner_block.open_parentheses += depth_change + + # Also check if we are starting or ending an inline assembly block. + if inner_block.inline_asm in (_NO_ASM, _END_ASM): + if (depth_change != 0 and inner_block.open_parentheses == 1 and + _MATCH_ASM.match(line)): + # Enter assembly block + inner_block.inline_asm = _INSIDE_ASM + else: + # Not entering assembly block. If previous line was _END_ASM, + # we will now shift to _NO_ASM state. + inner_block.inline_asm = _NO_ASM + elif (inner_block.inline_asm == _INSIDE_ASM and + inner_block.open_parentheses == 0): + # Exit assembly block + inner_block.inline_asm = _END_ASM + + # Consume namespace declaration at the beginning of the line. Do + # this in a loop so that we catch same line declarations like this: + # namespace proto2 { namespace bridge { class MessageSet; } } + while True: + # Match start of namespace. The "\b\s*" below catches namespace + # declarations even if it weren't followed by a whitespace, this + # is so that we don't confuse our namespace checker. The + # missing spaces will be flagged by CheckSpacing. + namespace_decl_match = Match(r'^\s*namespace\b\s*([:\w]+)?(.*)$', + line) + if not namespace_decl_match: + break + + new_namespace = _NamespaceInfo( + namespace_decl_match.group(1), linenum) + self.stack.append(new_namespace) + + line = namespace_decl_match.group(2) + if line.find('{') != -1: + new_namespace.seen_open_brace = True + line = line[line.find('{') + 1:] + + # Look for a class declaration in whatever is left of the line + # after parsing namespaces. The regexp accounts for decorated classes + # such as in: + # class LOCKABLE API Object { + # }; + class_decl_match = Match( + r'^(\s*(?:template\s*<[\w\s<>,:]*>\s*)?' + r'(class|struct)\s+(?:[A-Z_]+\s+)*(\w+(?:::\w+)*))' + r'(.*)$', line) + if (class_decl_match and + (not self.stack or self.stack[-1].open_parentheses == 0)): + # We do not want to accept classes that are actually template arguments: + # template , + # template class Ignore3> + # void Function() {}; + # + # To avoid template argument cases, we scan forward and look for + # an unmatched '>'. If we see one, assume we are inside a + # template argument list. + end_declaration = len(class_decl_match.group(1)) + if not self.InTemplateArgumentList(clean_lines, linenum, + end_declaration): + self.stack.append( + _ClassInfo( + class_decl_match.group(3), + class_decl_match.group(2), clean_lines, linenum)) + line = class_decl_match.group(4) + + # If we have not yet seen the opening brace for the innermost block, + # run checks here. + if not self.SeenOpenBrace(): + self.stack[-1].CheckBegin(filename, clean_lines, linenum, error) + + # Update access control if we are inside a class/struct + if self.stack and isinstance(self.stack[-1], _ClassInfo): + classinfo = self.stack[-1] + access_match = Match( + r'^(.*)\b(public|private|protected|signals)(\s+(?:slots\s*)?)?' + r':(?:[^:]|$)', line) + if access_match: + classinfo.access = access_match.group(2) + + # Check that access keywords are indented +1 space. Skip this + # check if the keywords are not preceded by whitespaces. + indent = access_match.group(1) + if (len(indent) != classinfo.class_indent + 1 and + Match(r'^\s*$', indent)): + if classinfo.is_struct: + parent = 'struct ' + classinfo.name + else: + parent = 'class ' + classinfo.name + slots = '' + if access_match.group(3): + slots = access_match.group(3) + error(filename, linenum, 'whitespace/indent', 3, + '%s%s: should be indented +1 space inside %s' % ( + access_match.group(2), slots, parent)) + + # Consume braces or semicolons from what's left of the line + while True: + # Match first brace, semicolon, or closed parenthesis. + matched = Match(r'^[^{;)}]*([{;)}])(.*)$', line) + if not matched: + break + + token = matched.group(1) + if token == '{': + # If namespace or class hasn't seen a opening brace yet, mark + # namespace/class head as complete. Push a new block onto the + # stack otherwise. + if not self.SeenOpenBrace(): + self.stack[-1].seen_open_brace = True + elif Match(r'^extern\s*"[^"]*"\s*\{', line): + self.stack.append(_ExternCInfo()) + else: + self.stack.append(_BlockInfo(True)) + if _MATCH_ASM.match(line): + self.stack[-1].inline_asm = _BLOCK_ASM + + elif token == ';' or token == ')': + # If we haven't seen an opening brace yet, but we already saw + # a semicolon, this is probably a forward declaration. Pop + # the stack for these. + # + # Similarly, if we haven't seen an opening brace yet, but we + # already saw a closing parenthesis, then these are probably + # function arguments with extra "class" or "struct" keywords. + # Also pop these stack for these. + if not self.SeenOpenBrace(): + self.stack.pop() + else: # token == '}' + # Perform end of block checks and pop the stack. + if self.stack: + self.stack[-1].CheckEnd(filename, clean_lines, linenum, + error) + self.stack.pop() + line = matched.group(2) + + def InnermostClass(self): + """Get class info on the top of the stack. Returns: A _ClassInfo object if we are inside a class, or None otherwise. """ - for i in range(len(self.stack), 0, -1): - classinfo = self.stack[i - 1] - if isinstance(classinfo, _ClassInfo): - return classinfo - return None + for i in range(len(self.stack), 0, -1): + classinfo = self.stack[i - 1] + if isinstance(classinfo, _ClassInfo): + return classinfo + return None - def CheckCompletedBlocks(self, filename, error): - """Checks that all classes and namespaces have been completely parsed. + def CheckCompletedBlocks(self, filename, error): + """Checks that all classes and namespaces have been completely parsed. Call this when all lines in a file have been processed. Args: filename: The name of the current file. error: The function to call with any errors found. """ - # Note: This test can result in false positives if #ifdef constructs - # get in the way of brace matching. See the testBuildClass test in - # cpplint_unittest.py for an example of this. - for obj in self.stack: - if isinstance(obj, _ClassInfo): - error(filename, obj.starting_linenum, 'build/class', 5, - 'Failed to find complete declaration of class %s' % - obj.name) - elif isinstance(obj, _NamespaceInfo): - error(filename, obj.starting_linenum, 'build/namespaces', 5, - 'Failed to find complete declaration of namespace %s' % - obj.name) - - -def CheckForNonStandardConstructs(filename, clean_lines, linenum, - nesting_state, error): - r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. + # Note: This test can result in false positives if #ifdef constructs + # get in the way of brace matching. See the testBuildClass test in + # cpplint_unittest.py for an example of this. + for obj in self.stack: + if isinstance(obj, _ClassInfo): + error(filename, obj.starting_linenum, 'build/class', 5, + 'Failed to find complete declaration of class %s' % + obj.name) + elif isinstance(obj, _NamespaceInfo): + error(filename, obj.starting_linenum, 'build/namespaces', 5, + 'Failed to find complete declaration of namespace %s' % + obj.name) + + +def CheckForNonStandardConstructs(filename, clean_lines, linenum, nesting_state, + error): + r"""Logs an error if we see certain non-ANSI constructs ignored by gcc-2. Complain about several constructs which gcc-2 accepts, but which are not standard C++. Warning about these in lint is one way to ease the @@ -2608,143 +2626,144 @@ def CheckForNonStandardConstructs(filename, clean_lines, linenum, filename, line number, error level, and message """ - # Remove comments from the line, but leave in strings for now. - line = clean_lines.lines[linenum] - - if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line): - error(filename, linenum, 'runtime/printf_format', 3, - '%q in format strings is deprecated. Use %ll instead.') - - if Search(r'printf\s*\(.*".*%\d+\$', line): - error(filename, linenum, 'runtime/printf_format', 2, - '%N$ formats are unconventional. Try rewriting to avoid them.') - - # Remove escaped backslashes before looking for undefined escapes. - line = line.replace('\\\\', '') - - if Search(r'("|\').*\\(%|\[|\(|{)', line): - error(filename, linenum, 'build/printf_format', 3, - '%, [, (, and { are undefined character escapes. Unescape them.') - - # For the rest, work with both comments and strings removed. - line = clean_lines.elided[linenum] - - if Search(r'\b(const|volatile|void|char|short|int|long' - r'|float|double|signed|unsigned' - r'|schar|u?int8|u?int16|u?int32|u?int64)' - r'\s+(register|static|extern|typedef)\b', - line): - error(filename, linenum, 'build/storage_class', 5, - 'Storage class (static, extern, typedef, etc) should be first.') - - if Match(r'\s*#\s*endif\s*[^/\s]+', line): - error(filename, linenum, 'build/endif_comment', 5, - 'Uncommented text after #endif is non-standard. Use a comment.') - - if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line): - error(filename, linenum, 'build/forward_decl', 5, - 'Inner-style forward declarations are invalid. Remove this line.') - - if Search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', - line): - error(filename, linenum, 'build/deprecated', 3, - '>? and ))?' - # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' - error(filename, linenum, 'runtime/member_string_references', 2, - 'const string& members are dangerous. It is much better to use ' - 'alternatives, such as pointers or simple constants.') - - # Everything else in this function operates on class declarations. - # Return early if the top of the nesting stack is not a class, or if - # the class head is not completed yet. - classinfo = nesting_state.InnermostClass() - if not classinfo or not classinfo.seen_open_brace: - return - - # The class may have been declared with namespace or classname qualifiers. - # The constructor and destructor will not have those qualifiers. - base_classname = classinfo.name.split('::')[-1] - - # Look for single-argument constructors that aren't marked explicit. - # Technically a valid construct, but against style. Also look for - # non-single-argument constructors which are also technically valid, but - # strongly suggest something is wrong. - explicit_constructor_match = Match( - r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*' - r'\(((?:[^()]|\([^()]*\))*)\)' - % re.escape(base_classname), - line) - - if explicit_constructor_match: - is_marked_explicit = explicit_constructor_match.group(1) - - if not explicit_constructor_match.group(2): - constructor_args = [] - else: - constructor_args = explicit_constructor_match.group(2).split(',') - - # collapse arguments so that commas in template parameter lists and function - # argument parameter lists don't split arguments in two - i = 0 - while i < len(constructor_args): - constructor_arg = constructor_args[i] - while (constructor_arg.count('<') > constructor_arg.count('>') or - constructor_arg.count('(') > constructor_arg.count(')')): - constructor_arg += ',' + constructor_args[i + 1] - del constructor_args[i + 1] - constructor_args[i] = constructor_arg - i += 1 - - defaulted_args = [arg for arg in constructor_args if '=' in arg] - noarg_constructor = (not constructor_args or # empty arg list - # 'void' arg specifier - (len(constructor_args) == 1 and - constructor_args[0].strip() == 'void')) - onearg_constructor = ((len(constructor_args) == 1 and # exactly one arg - not noarg_constructor) or - # all but at most one arg defaulted - (len(constructor_args) >= 1 and - not noarg_constructor and - len(defaulted_args) >= len(constructor_args) - 1)) - initializer_list_constructor = bool( - onearg_constructor and - Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) - copy_constructor = bool( - onearg_constructor and - Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' - % re.escape(base_classname), constructor_args[0].strip())) - - if (not is_marked_explicit and - onearg_constructor and - not initializer_list_constructor and - not copy_constructor): - if defaulted_args: - error(filename, linenum, 'runtime/explicit', 5, - 'Constructors callable with one argument ' - 'should be marked explicit.') - else: - error(filename, linenum, 'runtime/explicit', 5, - 'Single-parameter constructors should be marked explicit.') - elif is_marked_explicit and not onearg_constructor: - if noarg_constructor: - error(filename, linenum, 'runtime/explicit', 5, - 'Zero-parameter constructors should not be marked explicit.') - else: - error(filename, linenum, 'runtime/explicit', 0, - 'Constructors that require multiple arguments ' - 'should not be marked explicit.') + # Remove comments from the line, but leave in strings for now. + line = clean_lines.lines[linenum] + + if Search(r'printf\s*\(.*".*%[-+ ]?\d*q', line): + error(filename, linenum, 'runtime/printf_format', 3, + '%q in format strings is deprecated. Use %ll instead.') + + if Search(r'printf\s*\(.*".*%\d+\$', line): + error(filename, linenum, 'runtime/printf_format', 2, + '%N$ formats are unconventional. Try rewriting to avoid them.') + + # Remove escaped backslashes before looking for undefined escapes. + line = line.replace('\\\\', '') + + if Search(r'("|\').*\\(%|\[|\(|{)', line): + error(filename, linenum, 'build/printf_format', 3, + '%, [, (, and { are undefined character escapes. Unescape them.') + + # For the rest, work with both comments and strings removed. + line = clean_lines.elided[linenum] + + if Search(r'\b(const|volatile|void|char|short|int|long' + r'|float|double|signed|unsigned' + r'|schar|u?int8|u?int16|u?int32|u?int64)' + r'\s+(register|static|extern|typedef)\b', line): + error(filename, linenum, 'build/storage_class', 5, + 'Storage class (static, extern, typedef, etc) should be first.') + + if Match(r'\s*#\s*endif\s*[^/\s]+', line): + error(filename, linenum, 'build/endif_comment', 5, + 'Uncommented text after #endif is non-standard. Use a comment.') + + if Match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line): + error( + filename, linenum, 'build/forward_decl', 5, + 'Inner-style forward declarations are invalid. Remove this line.') + + if Search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', + line): + error( + filename, linenum, 'build/deprecated', 3, + '>? and ))?' + # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' + error(filename, linenum, 'runtime/member_string_references', 2, + 'const string& members are dangerous. It is much better to use ' + 'alternatives, such as pointers or simple constants.') + + # Everything else in this function operates on class declarations. + # Return early if the top of the nesting stack is not a class, or if + # the class head is not completed yet. + classinfo = nesting_state.InnermostClass() + if not classinfo or not classinfo.seen_open_brace: + return + + # The class may have been declared with namespace or classname qualifiers. + # The constructor and destructor will not have those qualifiers. + base_classname = classinfo.name.split('::')[-1] + + # Look for single-argument constructors that aren't marked explicit. + # Technically a valid construct, but against style. Also look for + # non-single-argument constructors which are also technically valid, but + # strongly suggest something is wrong. + explicit_constructor_match = Match( + r'\s+(?:inline\s+)?(explicit\s+)?(?:inline\s+)?%s\s*' + r'\(((?:[^()]|\([^()]*\))*)\)' % re.escape(base_classname), line) + + if explicit_constructor_match: + is_marked_explicit = explicit_constructor_match.group(1) + + if not explicit_constructor_match.group(2): + constructor_args = [] + else: + constructor_args = explicit_constructor_match.group(2).split(',') + + # collapse arguments so that commas in template parameter lists and function + # argument parameter lists don't split arguments in two + i = 0 + while i < len(constructor_args): + constructor_arg = constructor_args[i] + while (constructor_arg.count('<') > constructor_arg.count('>') or + constructor_arg.count('(') > constructor_arg.count(')')): + constructor_arg += ',' + constructor_args[i + 1] + del constructor_args[i + 1] + constructor_args[i] = constructor_arg + i += 1 + + defaulted_args = [arg for arg in constructor_args if '=' in arg] + noarg_constructor = ( + not constructor_args or # empty arg list + # 'void' arg specifier + (len(constructor_args) == 1 and + constructor_args[0].strip() == 'void')) + onearg_constructor = ( + ( + len(constructor_args) == 1 and # exactly one arg + not noarg_constructor) or + # all but at most one arg defaulted + (len(constructor_args) >= 1 and not noarg_constructor and + len(defaulted_args) >= len(constructor_args) - 1)) + initializer_list_constructor = bool( + onearg_constructor and + Search(r'\bstd\s*::\s*initializer_list\b', constructor_args[0])) + copy_constructor = bool( + onearg_constructor and + Match(r'(const\s+)?%s(\s*<[^>]*>)?(\s+const)?\s*(?:<\w+>\s*)?&' % + re.escape(base_classname), constructor_args[0].strip())) + + if (not is_marked_explicit and onearg_constructor and + not initializer_list_constructor and not copy_constructor): + if defaulted_args: + error(filename, linenum, 'runtime/explicit', 5, + 'Constructors callable with one argument ' + 'should be marked explicit.') + else: + error( + filename, linenum, 'runtime/explicit', 5, + 'Single-parameter constructors should be marked explicit.') + elif is_marked_explicit and not onearg_constructor: + if noarg_constructor: + error( + filename, linenum, 'runtime/explicit', 5, + 'Zero-parameter constructors should not be marked explicit.') + else: + error(filename, linenum, 'runtime/explicit', 0, + 'Constructors that require multiple arguments ' + 'should not be marked explicit.') def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): - """Checks for the correctness of various spacing around function calls. + """Checks for the correctness of various spacing around function calls. Args: filename: The name of the current file. @@ -2752,75 +2771,74 @@ def CheckSpacingForFunctionCall(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Since function calls often occur inside if/for/while/switch - # expressions - which have their own, more liberal conventions - we - # first see if we should be looking inside such an expression for a - # function call, to which we can apply more strict standards. - fncall = line # if there's no control flow construct, look at whole line - for pattern in (r'\bif\s*\((.*)\)\s*{', - r'\bfor\s*\((.*)\)\s*{', - r'\bwhile\s*\((.*)\)\s*[{;]', - r'\bswitch\s*\((.*)\)\s*{'): - match = Search(pattern, line) - if match: - fncall = match.group(1) # look inside the parens for function calls - break - - # Except in if/for/while/switch, there should never be space - # immediately inside parens (eg "f( 3, 4 )"). We make an exception - # for nested parens ( (a+b) + c ). Likewise, there should never be - # a space before a ( when it's a function argument. I assume it's a - # function argument when the char before the whitespace is legal in - # a function name (alnum + _) and we're not starting a macro. Also ignore - # pointers and references to arrays and functions coz they're too tricky: - # we use a very simple way to recognize these: - # " (something)(maybe-something)" or - # " (something)(maybe-something," or - # " (something)[something]" - # Note that we assume the contents of [] to be short enough that - # they'll never need to wrap. - if ( # Ignore control structures. - not Search(r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', - fncall) and - # Ignore pointers/references to functions. - not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and - # Ignore pointers/references to arrays. - not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): - if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space after ( in function call') - elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space after (') - if (Search(r'\w\s+\(', fncall) and - not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and - not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and - not Search(r'\bcase\s+\(', fncall)): - # TODO(unknown): Space after an operator function seem to be a common - # error, silence those for now by restricting them to highest verbosity. - if Search(r'\boperator_*\b', line): - error(filename, linenum, 'whitespace/parens', 0, - 'Extra space before ( in function call') - else: - error(filename, linenum, 'whitespace/parens', 4, - 'Extra space before ( in function call') - # If the ) is followed only by a newline or a { + newline, assume it's - # part of a control statement (if/while/etc), and don't complain - if Search(r'[^)]\s+\)\s*[^{\s]', fncall): - # If the closing parenthesis is preceded by only whitespaces, - # try to give a more descriptive error message. - if Search(r'^\s+\)', fncall): - error(filename, linenum, 'whitespace/parens', 2, - 'Closing ) should be moved to the previous line') - else: - error(filename, linenum, 'whitespace/parens', 2, - 'Extra space before )') + line = clean_lines.elided[linenum] + + # Since function calls often occur inside if/for/while/switch + # expressions - which have their own, more liberal conventions - we + # first see if we should be looking inside such an expression for a + # function call, to which we can apply more strict standards. + fncall = line # if there's no control flow construct, look at whole line + for pattern in (r'\bif\s*\((.*)\)\s*{', r'\bfor\s*\((.*)\)\s*{', + r'\bwhile\s*\((.*)\)\s*[{;]', r'\bswitch\s*\((.*)\)\s*{'): + match = Search(pattern, line) + if match: + fncall = match.group(1) # look inside the parens for function calls + break + + # Except in if/for/while/switch, there should never be space + # immediately inside parens (eg "f( 3, 4 )"). We make an exception + # for nested parens ( (a+b) + c ). Likewise, there should never be + # a space before a ( when it's a function argument. I assume it's a + # function argument when the char before the whitespace is legal in + # a function name (alnum + _) and we're not starting a macro. Also ignore + # pointers and references to arrays and functions coz they're too tricky: + # we use a very simple way to recognize these: + # " (something)(maybe-something)" or + # " (something)(maybe-something," or + # " (something)[something]" + # Note that we assume the contents of [] to be short enough that + # they'll never need to wrap. + if ( # Ignore control structures. + not Search( + r'\b(if|for|while|switch|return|new|delete|catch|sizeof)\b', + fncall) and + # Ignore pointers/references to functions. + not Search(r' \([^)]+\)\([^)]*(\)|,$)', fncall) and + # Ignore pointers/references to arrays. + not Search(r' \([^)]+\)\[[^\]]+\]', fncall)): + if Search(r'\w\s*\(\s(?!\s*\\$)', fncall): # a ( used for a fn call + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space after ( in function call') + elif Search(r'\(\s+(?!(\s*\\)|\()', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space after (') + if (Search(r'\w\s+\(', fncall) and + not Search(r'#\s*define|typedef|using\s+\w+\s*=', fncall) and + not Search(r'\w\s+\((\w+::)*\*\w+\)\(', fncall) and + not Search(r'\bcase\s+\(', fncall)): + # TODO(unknown): Space after an operator function seem to be a common + # error, silence those for now by restricting them to highest verbosity. + if Search(r'\boperator_*\b', line): + error(filename, linenum, 'whitespace/parens', 0, + 'Extra space before ( in function call') + else: + error(filename, linenum, 'whitespace/parens', 4, + 'Extra space before ( in function call') + # If the ) is followed only by a newline or a { + newline, assume it's + # part of a control statement (if/while/etc), and don't complain + if Search(r'[^)]\s+\)\s*[^{\s]', fncall): + # If the closing parenthesis is preceded by only whitespaces, + # try to give a more descriptive error message. + if Search(r'^\s+\)', fncall): + error(filename, linenum, 'whitespace/parens', 2, + 'Closing ) should be moved to the previous line') + else: + error(filename, linenum, 'whitespace/parens', 2, + 'Extra space before )') def IsBlankLine(line): - """Returns true if the given line is blank. + """Returns true if the given line is blank. We consider a line to be blank if the line is empty or consists of only white spaces. @@ -2831,26 +2849,26 @@ def IsBlankLine(line): Returns: True, if the given line is blank. """ - return not line or line.isspace() + return not line or line.isspace() def CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, error): - is_namespace_indent_item = ( - len(nesting_state.stack) > 1 and - nesting_state.stack[-1].check_namespace_indentation and - isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and - nesting_state.previous_stack_top == nesting_state.stack[-2]) + is_namespace_indent_item = ( + len(nesting_state.stack) > 1 and + nesting_state.stack[-1].check_namespace_indentation and + isinstance(nesting_state.previous_stack_top, _NamespaceInfo) and + nesting_state.previous_stack_top == nesting_state.stack[-2]) - if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, - clean_lines.elided, line): - CheckItemIndentationInNamespace(filename, clean_lines.elided, - line, error) + if ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, + clean_lines.elided, line): + CheckItemIndentationInNamespace(filename, clean_lines.elided, line, + error) -def CheckForFunctionLengths(filename, clean_lines, linenum, - function_state, error): - """Reports for long function bodies. +def CheckForFunctionLengths(filename, clean_lines, linenum, function_state, + error): + """Reports for long function bodies. For an overview why this is done, see: http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions @@ -2871,56 +2889,57 @@ def CheckForFunctionLengths(filename, clean_lines, linenum, function_state: Current function name and lines in body so far. error: The function to call with any errors found. """ - lines = clean_lines.lines - line = lines[linenum] - joined_line = '' - - starting_func = False - regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... - match_result = Match(regexp, line) - if match_result: - # If the name is all caps and underscores, figure it's a macro and - # ignore it, unless it's TEST or TEST_F. - function_name = match_result.group(1).split()[-1] - if function_name == 'TEST' or function_name == 'TEST_F' or ( - not Match(r'[A-Z_]+$', function_name)): - starting_func = True - - if starting_func: - body_found = False - for start_linenum in xrange(linenum, clean_lines.NumLines()): - start_line = lines[start_linenum] - joined_line += ' ' + start_line.lstrip() - if Search(r'(;|})', start_line): # Declarations and trivial functions - body_found = True - break # ... ignore - elif Search(r'{', start_line): - body_found = True - function = Search(r'((\w|:)*)\(', line).group(1) - if Match(r'TEST', function): # Handle TEST... macros - parameter_regexp = Search(r'(\(.*\))', joined_line) - if parameter_regexp: # Ignore bad syntax - function += parameter_regexp.group(1) - else: - function += '()' - function_state.Begin(function) - break - if not body_found: - # No body for the function (or evidence of a non-function) was found. - error(filename, linenum, 'readability/fn_size', 5, - 'Lint failed to find start of function body.') - elif Match(r'^\}\s*$', line): # function end - function_state.Check(error, filename, linenum) - function_state.End() - elif not Match(r'^\s*$', line): - function_state.Count() # Count non-blank/non-comment lines. + lines = clean_lines.lines + line = lines[linenum] + joined_line = '' + + starting_func = False + regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... + match_result = Match(regexp, line) + if match_result: + # If the name is all caps and underscores, figure it's a macro and + # ignore it, unless it's TEST or TEST_F. + function_name = match_result.group(1).split()[-1] + if function_name == 'TEST' or function_name == 'TEST_F' or ( + not Match(r'[A-Z_]+$', function_name)): + starting_func = True + + if starting_func: + body_found = False + for start_linenum in xrange(linenum, clean_lines.NumLines()): + start_line = lines[start_linenum] + joined_line += ' ' + start_line.lstrip() + if Search(r'(;|})', + start_line): # Declarations and trivial functions + body_found = True + break # ... ignore + elif Search(r'{', start_line): + body_found = True + function = Search(r'((\w|:)*)\(', line).group(1) + if Match(r'TEST', function): # Handle TEST... macros + parameter_regexp = Search(r'(\(.*\))', joined_line) + if parameter_regexp: # Ignore bad syntax + function += parameter_regexp.group(1) + else: + function += '()' + function_state.Begin(function) + break + if not body_found: + # No body for the function (or evidence of a non-function) was found. + error(filename, linenum, 'readability/fn_size', 5, + 'Lint failed to find start of function body.') + elif Match(r'^\}\s*$', line): # function end + function_state.Check(error, filename, linenum) + function_state.End() + elif not Match(r'^\s*$', line): + function_state.Count() # Count non-blank/non-comment lines. _RE_PATTERN_TODO = re.compile(r'^//(\s*)TODO(\(.+?\))?:?(\s|$)?') def CheckComment(line, filename, linenum, next_line_start, error): - """Checks for common mistakes in comments. + """Checks for common mistakes in comments. Args: line: The line in question. @@ -2929,54 +2948,54 @@ def CheckComment(line, filename, linenum, next_line_start, error): next_line_start: The first non-whitespace column of the next line. error: The function to call with any errors found. """ - commentpos = line.find('//') - if commentpos != -1: - # Check if the // may be in quotes. If so, ignore it - # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison - if (line.count('"', 0, commentpos) - - line.count('\\"', 0, commentpos)) % 2 == 0: # not in quotes - # Allow one space for new scopes, two spaces otherwise: - if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) and - ((commentpos >= 1 and - line[commentpos-1] not in string.whitespace) or - (commentpos >= 2 and - line[commentpos-2] not in string.whitespace))): - error(filename, linenum, 'whitespace/comments', 2, - 'At least two spaces is best between code and comments') - - # Checks for common mistakes in TODO comments. - comment = line[commentpos:] - match = _RE_PATTERN_TODO.match(comment) - if match: - # One whitespace is correct; zero whitespace is handled elsewhere. - leading_whitespace = match.group(1) - if len(leading_whitespace) > 1: - error(filename, linenum, 'whitespace/todo', 2, - 'Too many spaces before TODO') - - username = match.group(2) - if not username: - error(filename, linenum, 'readability/todo', 2, - 'Missing username in TODO; it should look like ' - '"// TODO(my_username): Stuff."') - - middle_whitespace = match.group(3) - # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison - if middle_whitespace != ' ' and middle_whitespace != '': - error(filename, linenum, 'whitespace/todo', 2, - 'TODO(my_username) should be followed by a space') - - # If the comment contains an alphanumeric character, there - # should be a space somewhere between it and the // unless - # it's a /// or //! Doxygen comment. - if (Match(r'//[^ ]*\w', comment) and - not Match(r'(///|//\!)(\s+|$)', comment)): - error(filename, linenum, 'whitespace/comments', 4, - 'Should have a space between // and comment') + commentpos = line.find('//') + if commentpos != -1: + # Check if the // may be in quotes. If so, ignore it + # Comparisons made explicit for clarity -- pylint: disable=g-explicit-bool-comparison + if (line.count('"', 0, commentpos) - line.count('\\"', 0, commentpos) + ) % 2 == 0: # not in quotes + # Allow one space for new scopes, two spaces otherwise: + if (not (Match(r'^.*{ *//', line) and next_line_start == commentpos) + and ((commentpos >= 1 and + line[commentpos - 1] not in string.whitespace) or + (commentpos >= 2 and + line[commentpos - 2] not in string.whitespace))): + error(filename, linenum, 'whitespace/comments', 2, + 'At least two spaces is best between code and comments') + + # Checks for common mistakes in TODO comments. + comment = line[commentpos:] + match = _RE_PATTERN_TODO.match(comment) + if match: + # One whitespace is correct; zero whitespace is handled elsewhere. + leading_whitespace = match.group(1) + if len(leading_whitespace) > 1: + error(filename, linenum, 'whitespace/todo', 2, + 'Too many spaces before TODO') + + username = match.group(2) + if not username: + error(filename, linenum, 'readability/todo', 2, + 'Missing username in TODO; it should look like ' + '"// TODO(my_username): Stuff."') + + middle_whitespace = match.group(3) + # Comparisons made explicit for correctness -- pylint: disable=g-explicit-bool-comparison + if middle_whitespace != ' ' and middle_whitespace != '': + error(filename, linenum, 'whitespace/todo', 2, + 'TODO(my_username) should be followed by a space') + + # If the comment contains an alphanumeric character, there + # should be a space somewhere between it and the // unless + # it's a /// or //! Doxygen comment. + if (Match(r'//[^ ]*\w', comment) and + not Match(r'(///|//\!)(\s+|$)', comment)): + error(filename, linenum, 'whitespace/comments', 4, + 'Should have a space between // and comment') def CheckAccess(filename, clean_lines, linenum, nesting_state, error): - """Checks for improper use of DISALLOW* macros. + """Checks for improper use of DISALLOW* macros. Args: filename: The name of the current file. @@ -2986,27 +3005,27 @@ def CheckAccess(filename, clean_lines, linenum, nesting_state, error): the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] # get rid of comments and strings - - matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|' - r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line) - if not matched: - return - if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo): - if nesting_state.stack[-1].access != 'private': - error(filename, linenum, 'readability/constructors', 3, - '%s must be in the private: section' % matched.group(1)) - - else: - # Found DISALLOW* macro outside a class declaration, or perhaps it - # was used inside a function when it should have been part of the - # class declaration. We could issue a warning here, but it - # probably resulted in a compiler error already. - pass + line = clean_lines.elided[linenum] # get rid of comments and strings + + matched = Match((r'\s*(DISALLOW_COPY_AND_ASSIGN|' + r'DISALLOW_IMPLICIT_CONSTRUCTORS)'), line) + if not matched: + return + if nesting_state.stack and isinstance(nesting_state.stack[-1], _ClassInfo): + if nesting_state.stack[-1].access != 'private': + error(filename, linenum, 'readability/constructors', 3, + '%s must be in the private: section' % matched.group(1)) + + else: + # Found DISALLOW* macro outside a class declaration, or perhaps it + # was used inside a function when it should have been part of the + # class declaration. We could issue a warning here, but it + # probably resulted in a compiler error already. + pass def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): - """Checks for the correctness of various spacing issues in the code. + """Checks for the correctness of various spacing issues in the code. Things we check for: spaces around operators, spaces after if/for/while/switch, no spaces around parens in function calls, two @@ -3023,118 +3042,114 @@ def CheckSpacing(filename, clean_lines, linenum, nesting_state, error): error: The function to call with any errors found. """ - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw = clean_lines.lines_without_raw_strings - line = raw[linenum] - - # Before nixing comments, check if the line is blank for no good - # reason. This includes the first line after a block is opened, and - # blank lines at the end of a function (ie, right before a line like '}' - # - # Skip all the blank line checks if we are immediately inside a - # namespace body. In other words, don't issue blank line warnings - # for this block: - # namespace { - # - # } - # - # A warning about missing end of namespace comments will be issued instead. - # - # Also skip blank line checks for 'extern "C"' blocks, which are formatted - # like namespaces. - if (IsBlankLine(line) and - not nesting_state.InNamespaceBody() and - not nesting_state.InExternC()): - elided = clean_lines.elided - prev_line = elided[linenum - 1] - prevbrace = prev_line.rfind('{') - # TODO(unknown): Don't complain if line before blank line, and line after, - # both start with alnums and are indented the same amount. - # This ignores whitespace at the start of a namespace block - # because those are not usually indented. - if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: - # OK, we have a blank line at the start of a code block. Before we - # complain, we check if it is an exception to the rule: The previous - # non-empty line has the parameters of a function header that are indented - # 4 spaces (because they did not fit in a 80 column line when placed on - # the same line as the function name). We also check for the case where - # the previous line is indented 6 spaces, which may happen when the - # initializers of a constructor do not fit into a 80 column line. - exception = False - if Match(r' {6}\w', prev_line): # Initializer list? - # We are looking for the opening column of initializer list, which - # should be indented 4 spaces to cause 6 space indentation afterwards. - search_position = linenum-2 - while (search_position >= 0 - and Match(r' {6}\w', elided[search_position])): - search_position -= 1 - exception = (search_position >= 0 - and elided[search_position][:5] == ' :') - else: - # Search for the function arguments or an initializer list. We use a - # simple heuristic here: If the line is indented 4 spaces; and we have a - # closing paren, without the opening paren, followed by an opening brace - # or colon (for initializer lists) we assume that it is the last line of - # a function header. If we have a colon indented 4 spaces, it is an - # initializer list. - exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', - prev_line) - or Match(r' {4}:', prev_line)) - - if not exception: - error(filename, linenum, 'whitespace/blank_line', 2, - 'Redundant blank line at the start of a code block ' - 'should be deleted.') - # Ignore blank lines at the end of a block in a long if-else - # chain, like this: - # if (condition1) { - # // Something followed by a blank line + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw = clean_lines.lines_without_raw_strings + line = raw[linenum] + + # Before nixing comments, check if the line is blank for no good + # reason. This includes the first line after a block is opened, and + # blank lines at the end of a function (ie, right before a line like '}' + # + # Skip all the blank line checks if we are immediately inside a + # namespace body. In other words, don't issue blank line warnings + # for this block: + # namespace { # - # } else if (condition2) { - # // Something else # } + # + # A warning about missing end of namespace comments will be issued instead. + # + # Also skip blank line checks for 'extern "C"' blocks, which are formatted + # like namespaces. + if (IsBlankLine(line) and not nesting_state.InNamespaceBody() and + not nesting_state.InExternC()): + elided = clean_lines.elided + prev_line = elided[linenum - 1] + prevbrace = prev_line.rfind('{') + # TODO(unknown): Don't complain if line before blank line, and line after, + # both start with alnums and are indented the same amount. + # This ignores whitespace at the start of a namespace block + # because those are not usually indented. + if prevbrace != -1 and prev_line[prevbrace:].find('}') == -1: + # OK, we have a blank line at the start of a code block. Before we + # complain, we check if it is an exception to the rule: The previous + # non-empty line has the parameters of a function header that are indented + # 4 spaces (because they did not fit in a 80 column line when placed on + # the same line as the function name). We also check for the case where + # the previous line is indented 6 spaces, which may happen when the + # initializers of a constructor do not fit into a 80 column line. + exception = False + if Match(r' {6}\w', prev_line): # Initializer list? + # We are looking for the opening column of initializer list, which + # should be indented 4 spaces to cause 6 space indentation afterwards. + search_position = linenum - 2 + while (search_position >= 0 and + Match(r' {6}\w', elided[search_position])): + search_position -= 1 + exception = (search_position >= 0 and + elided[search_position][:5] == ' :') + else: + # Search for the function arguments or an initializer list. We use a + # simple heuristic here: If the line is indented 4 spaces; and we have a + # closing paren, without the opening paren, followed by an opening brace + # or colon (for initializer lists) we assume that it is the last line of + # a function header. If we have a colon indented 4 spaces, it is an + # initializer list. + exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', + prev_line) or Match(r' {4}:', prev_line)) + + if not exception: + error(filename, linenum, 'whitespace/blank_line', 2, + 'Redundant blank line at the start of a code block ' + 'should be deleted.') + # Ignore blank lines at the end of a block in a long if-else + # chain, like this: + # if (condition1) { + # // Something followed by a blank line + # + # } else if (condition2) { + # // Something else + # } + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + if (next_line and Match(r'\s*}', next_line) and + next_line.find('} else ') == -1): + error(filename, linenum, 'whitespace/blank_line', 3, + 'Redundant blank line at the end of a code block ' + 'should be deleted.') + + matched = Match(r'\s*(public|protected|private):', prev_line) + if matched: + error(filename, linenum, 'whitespace/blank_line', 3, + 'Do not leave a blank line after "%s:"' % matched.group(1)) + + # Next, check comments + next_line_start = 0 if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - if (next_line - and Match(r'\s*}', next_line) - and next_line.find('} else ') == -1): - error(filename, linenum, 'whitespace/blank_line', 3, - 'Redundant blank line at the end of a code block ' - 'should be deleted.') - - matched = Match(r'\s*(public|protected|private):', prev_line) - if matched: - error(filename, linenum, 'whitespace/blank_line', 3, - 'Do not leave a blank line after "%s:"' % matched.group(1)) - - # Next, check comments - next_line_start = 0 - if linenum + 1 < clean_lines.NumLines(): - next_line = raw[linenum + 1] - next_line_start = len(next_line) - len(next_line.lstrip()) - CheckComment(line, filename, linenum, next_line_start, error) + next_line = raw[linenum + 1] + next_line_start = len(next_line) - len(next_line.lstrip()) + CheckComment(line, filename, linenum, next_line_start, error) - # get rid of comments and strings - line = clean_lines.elided[linenum] + # get rid of comments and strings + line = clean_lines.elided[linenum] - # You shouldn't have spaces before your brackets, except maybe after - # 'delete []' or 'return []() {};' - if Search(r'\w\s+\[', line) and not Search(r'(?:delete|return)\s+\[', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Extra space before [') + # You shouldn't have spaces before your brackets, except maybe after + # 'delete []' or 'return []() {};' + if Search(r'\w\s+\[', line) and not Search(r'(?:delete|return)\s+\[', line): + error(filename, linenum, 'whitespace/braces', 5, 'Extra space before [') - # In range-based for, we wanted spaces before and after the colon, but - # not around "::" tokens that might appear. - if (Search(r'for *\(.*[^:]:[^: ]', line) or - Search(r'for *\(.*[^: ]:[^:]', line)): - error(filename, linenum, 'whitespace/forcolon', 2, - 'Missing space around colon in range-based for loop') + # In range-based for, we wanted spaces before and after the colon, but + # not around "::" tokens that might appear. + if (Search(r'for *\(.*[^:]:[^: ]', line) or + Search(r'for *\(.*[^: ]:[^:]', line)): + error(filename, linenum, 'whitespace/forcolon', 2, + 'Missing space around colon in range-based for loop') def CheckOperatorSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing around operators. + """Checks for horizontal spacing around operators. Args: filename: The name of the current file. @@ -3142,114 +3157,116 @@ def CheckOperatorSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Don't try to do spacing checks for operator methods. Do this by - # replacing the troublesome characters with something else, - # preserving column position for all other characters. - # - # The replacement is done repeatedly to avoid false positives from - # operators that call operators. - while True: - match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) - if match: - line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) - else: - break - - # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". - # Otherwise not. Note we only check for non-spaces on *both* sides; - # sometimes people put non-spaces on one side when aligning ='s among - # many lines (not that this is behavior that I approve of...) - if ((Search(r'[\w.]=', line) or - Search(r'=[\w.]', line)) - and not Search(r'\b(if|while|for) ', line) - # Operators taken from [lex.operators] in C++11 standard. - and not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) - and not Search(r'operator=', line)): - error(filename, linenum, 'whitespace/operators', 4, - 'Missing spaces around =') - - # It's ok not to have spaces around binary operators like + - * /, but if - # there's too little whitespace, we get concerned. It's hard to tell, - # though, so we punt on this one for now. TODO. - - # You should always have whitespace around binary operators. - # - # Check <= and >= first to avoid false positives with < and >, then - # check non-include lines for spacing around < and >. - # - # If the operator is followed by a comma, assume it's be used in a - # macro context and don't do any checks. This avoids false - # positives. - # - # Note that && is not included here. Those are checked separately - # in CheckRValueReference - match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around %s' % match.group(1)) - elif not Match(r'#.*include', line): - # Look for < that is not surrounded by spaces. This is only - # triggered if both sides are missing spaces, even though - # technically should should flag if at least one side is missing a - # space. This is done to avoid some false positives with shifts. - match = Match(r'^(.*[^\s<])<[^\s=<,]', line) + line = clean_lines.elided[linenum] + + # Don't try to do spacing checks for operator methods. Do this by + # replacing the troublesome characters with something else, + # preserving column position for all other characters. + # + # The replacement is done repeatedly to avoid false positives from + # operators that call operators. + while True: + match = Match(r'^(.*\boperator\b)(\S+)(\s*\(.*)$', line) + if match: + line = match.group(1) + ('_' * len(match.group(2))) + match.group(3) + else: + break + + # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". + # Otherwise not. Note we only check for non-spaces on *both* sides; + # sometimes people put non-spaces on one side when aligning ='s among + # many lines (not that this is behavior that I approve of...) + if ((Search(r'[\w.]=', line) or + Search(r'=[\w.]', line)) and not Search(r'\b(if|while|for) ', line) + # Operators taken from [lex.operators] in C++11 standard. + and + not Search(r'(>=|<=|==|!=|&=|\^=|\|=|\+=|\*=|\/=|\%=)', line) and + not Search(r'operator=', line)): + error(filename, linenum, 'whitespace/operators', 4, + 'Missing spaces around =') + + # It's ok not to have spaces around binary operators like + - * /, but if + # there's too little whitespace, we get concerned. It's hard to tell, + # though, so we punt on this one for now. TODO. + + # You should always have whitespace around binary operators. + # + # Check <= and >= first to avoid false positives with < and >, then + # check non-include lines for spacing around < and >. + # + # If the operator is followed by a comma, assume it's be used in a + # macro context and don't do any checks. This avoids false + # positives. + # + # Note that && is not included here. Those are checked separately + # in CheckRValueReference + match = Search(r'[^<>=!\s](==|!=|<=|>=|\|\|)[^<>=!\s,;\)]', line) if match: - (_, _, end_pos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if end_pos <= -1: error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <') + 'Missing spaces around %s' % match.group(1)) + elif not Match(r'#.*include', line): + # Look for < that is not surrounded by spaces. This is only + # triggered if both sides are missing spaces, even though + # technically should should flag if at least one side is missing a + # space. This is done to avoid some false positives with shifts. + match = Match(r'^(.*[^\s<])<[^\s=<,]', line) + if match: + (_, _, end_pos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if end_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <') + + # Look for > that is not surrounded by spaces. Similar to the + # above, we only trigger if both sides are missing spaces to avoid + # false positives with shifts. + match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) + if match: + (_, _, start_pos) = ReverseCloseExpression(clean_lines, linenum, + len(match.group(1))) + if start_pos <= -1: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around >') + + # We allow no-spaces around << when used like this: 10<<20, but + # not otherwise (particularly, not when used as streams) + # + # We also allow operators following an opening parenthesis, since + # those tend to be macros that deal with operators. + match = Search(r'(operator|[^\s(<])(?:L|UL|ULL|l|ul|ull)?<<([^\s,=<])', + line) + if (match and + not (match.group(1).isdigit() and match.group(2).isdigit()) and + not (match.group(1) == 'operator' and match.group(2) == ';')): + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around <<') - # Look for > that is not surrounded by spaces. Similar to the - # above, we only trigger if both sides are missing spaces to avoid - # false positives with shifts. - match = Match(r'^(.*[^-\s>])>[^\s=>,]', line) + # We allow no-spaces around >> for almost anything. This is because + # C++11 allows ">>" to close nested templates, which accounts for + # most cases when ">>" is not followed by a space. + # + # We still warn on ">>" followed by alpha character, because that is + # likely due to ">>" being used for right shifts, e.g.: + # value >> alpha + # + # When ">>" is used to close templates, the alphanumeric letter that + # follows would be part of an identifier, and there should still be + # a space separating the template type and the identifier. + # type> alpha + match = Search(r'>>[a-zA-Z_]', line) if match: - (_, _, start_pos) = ReverseCloseExpression( - clean_lines, linenum, len(match.group(1))) - if start_pos <= -1: error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >') - - # We allow no-spaces around << when used like this: 10<<20, but - # not otherwise (particularly, not when used as streams) - # - # We also allow operators following an opening parenthesis, since - # those tend to be macros that deal with operators. - match = Search(r'(operator|[^\s(<])(?:L|UL|ULL|l|ul|ull)?<<([^\s,=<])', line) - if (match and not (match.group(1).isdigit() and match.group(2).isdigit()) and - not (match.group(1) == 'operator' and match.group(2) == ';')): - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around <<') - - # We allow no-spaces around >> for almost anything. This is because - # C++11 allows ">>" to close nested templates, which accounts for - # most cases when ">>" is not followed by a space. - # - # We still warn on ">>" followed by alpha character, because that is - # likely due to ">>" being used for right shifts, e.g.: - # value >> alpha - # - # When ">>" is used to close templates, the alphanumeric letter that - # follows would be part of an identifier, and there should still be - # a space separating the template type and the identifier. - # type> alpha - match = Search(r'>>[a-zA-Z_]', line) - if match: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around >>') - - # There shouldn't be space around unary operators - match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) - if match: - error(filename, linenum, 'whitespace/operators', 4, - 'Extra space for operator %s' % match.group(1)) + 'Missing spaces around >>') + + # There shouldn't be space around unary operators + match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) + if match: + error(filename, linenum, 'whitespace/operators', 4, + 'Extra space for operator %s' % match.group(1)) def CheckParenthesisSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing around parentheses. + """Checks for horizontal spacing around parentheses. Args: filename: The name of the current file. @@ -3257,37 +3274,36 @@ def CheckParenthesisSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # No spaces after an if, while, switch, or for - match = Search(r' (if\(|for\(|while\(|switch\()', line) - if match: - error(filename, linenum, 'whitespace/parens', 5, - 'Missing space before ( in %s' % match.group(1)) - - # For if/for/while/switch, the left and right parens should be - # consistent about how many spaces are inside the parens, and - # there should either be zero or one spaces inside the parens. - # We don't want: "if ( foo)" or "if ( foo )". - # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. - match = Search(r'\b(if|for|while|switch)\s*' - r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', - line) - if match: - if len(match.group(2)) != len(match.group(4)): - if not (match.group(3) == ';' and - len(match.group(2)) == 1 + len(match.group(4)) or - not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + line = clean_lines.elided[linenum] + + # No spaces after an if, while, switch, or for + match = Search(r' (if\(|for\(|while\(|switch\()', line) + if match: error(filename, linenum, 'whitespace/parens', 5, - 'Mismatching spaces inside () in %s' % match.group(1)) - if len(match.group(2)) not in [0, 1]: - error(filename, linenum, 'whitespace/parens', 5, - 'Should have zero or one spaces inside ( and ) in %s' % - match.group(1)) + 'Missing space before ( in %s' % match.group(1)) + + # For if/for/while/switch, the left and right parens should be + # consistent about how many spaces are inside the parens, and + # there should either be zero or one spaces inside the parens. + # We don't want: "if ( foo)" or "if ( foo )". + # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. + match = Search(r'\b(if|for|while|switch)\s*' + r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', line) + if match: + if len(match.group(2)) != len(match.group(4)): + if not (match.group(3) == ';' and + len(match.group(2)) == 1 + len(match.group(4)) or + not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + error(filename, linenum, 'whitespace/parens', 5, + 'Mismatching spaces inside () in %s' % match.group(1)) + if len(match.group(2)) not in [0, 1]: + error(filename, linenum, 'whitespace/parens', 5, + 'Should have zero or one spaces inside ( and ) in %s' % + match.group(1)) def CheckCommaSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing near commas and semicolons. + """Checks for horizontal spacing near commas and semicolons. Args: filename: The name of the current file. @@ -3295,35 +3311,34 @@ def CheckCommaSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - raw = clean_lines.lines_without_raw_strings - line = clean_lines.elided[linenum] - - # You should always have a space after a comma (either as fn arg or operator) - # - # This does not apply when the non-space character following the - # comma is another comma, since the only time when that happens is - # for empty macro arguments. - # - # We run this check in two passes: first pass on elided lines to - # verify that lines contain missing whitespaces, second pass on raw - # lines to confirm that those missing whitespaces are not due to - # elided comments. - if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and - Search(r',[^,\s]', raw[linenum])): - error(filename, linenum, 'whitespace/comma', 3, - 'Missing space after ,') - - # You should always have a space after a semicolon - # except for few corner cases - # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more - # space after ; - if Search(r';[^\s};\\)/]', line): - error(filename, linenum, 'whitespace/semicolon', 3, - 'Missing space after ;') + raw = clean_lines.lines_without_raw_strings + line = clean_lines.elided[linenum] + + # You should always have a space after a comma (either as fn arg or operator) + # + # This does not apply when the non-space character following the + # comma is another comma, since the only time when that happens is + # for empty macro arguments. + # + # We run this check in two passes: first pass on elided lines to + # verify that lines contain missing whitespaces, second pass on raw + # lines to confirm that those missing whitespaces are not due to + # elided comments. + if (Search(r',[^,\s]', ReplaceAll(r'\boperator\s*,\s*\(', 'F(', line)) and + Search(r',[^,\s]', raw[linenum])): + error(filename, linenum, 'whitespace/comma', 3, 'Missing space after ,') + + # You should always have a space after a semicolon + # except for few corner cases + # TODO(unknown): clarify if 'if (1) { return 1;}' is requires one more + # space after ; + if Search(r';[^\s};\\)/]', line): + error(filename, linenum, 'whitespace/semicolon', 3, + 'Missing space after ;') def CheckBracesSpacing(filename, clean_lines, linenum, error): - """Checks for horizontal spacing near commas. + """Checks for horizontal spacing near commas. Args: filename: The name of the current file. @@ -3331,78 +3346,78 @@ def CheckBracesSpacing(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Except after an opening paren, or after another opening brace (in case of - # an initializer list, for instance), you should have spaces before your - # braces. And since you should never have braces at the beginning of a line, - # this is an easy test. - match = Match(r'^(.*[^ ({>]){', line) - if match: - # Try a bit harder to check for brace initialization. This - # happens in one of the following forms: - # Constructor() : initializer_list_{} { ... } - # Constructor{}.MemberFunction() - # Type variable{}; - # FunctionCall(type{}, ...); - # LastArgument(..., type{}); - # LOG(INFO) << type{} << " ..."; - # map_of_type[{...}] = ...; - # ternary = expr ? new type{} : nullptr; - # OuterTemplate{}> - # - # We check for the character following the closing brace, and - # silence the warning if it's one of those listed above, i.e. - # "{.;,)<>]:". - # - # To account for nested initializer list, we allow any number of - # closing braces up to "{;,)<". We can't simply silence the - # warning on first sight of closing brace, because that would - # cause false negatives for things that are not initializer lists. - # Silence this: But not this: - # Outer{ if (...) { - # Inner{...} if (...){ // Missing space before { - # }; } - # - # There is a false negative with this approach if people inserted - # spurious semicolons, e.g. "if (cond){};", but we will catch the - # spurious semicolon with a separate check. - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - trailing_text = '' - if endpos > -1: - trailing_text = endline[endpos:] - for offset in xrange(endlinenum + 1, - min(endlinenum + 3, clean_lines.NumLines() - 1)): - trailing_text += clean_lines.elided[offset] - if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before {') - - # Make sure '} else {' has spaces. - if Search(r'}else', line): - error(filename, linenum, 'whitespace/braces', 5, - 'Missing space before else') - - # You shouldn't have a space before a semicolon at the end of the line. - # There's a special case for "for" since the style guide allows space before - # the semicolon there. - if Search(r':\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Semicolon defining empty statement. Use {} instead.') - elif Search(r'^\s*;\s*$', line): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Line contains only semicolon. If this should be an empty statement, ' - 'use {} instead.') - elif (Search(r'\s+;\s*$', line) and - not Search(r'\bfor\b', line)): - error(filename, linenum, 'whitespace/semicolon', 5, - 'Extra space before last semicolon. If this should be an empty ' - 'statement, use {} instead.') + line = clean_lines.elided[linenum] + + # Except after an opening paren, or after another opening brace (in case of + # an initializer list, for instance), you should have spaces before your + # braces. And since you should never have braces at the beginning of a line, + # this is an easy test. + match = Match(r'^(.*[^ ({>]){', line) + if match: + # Try a bit harder to check for brace initialization. This + # happens in one of the following forms: + # Constructor() : initializer_list_{} { ... } + # Constructor{}.MemberFunction() + # Type variable{}; + # FunctionCall(type{}, ...); + # LastArgument(..., type{}); + # LOG(INFO) << type{} << " ..."; + # map_of_type[{...}] = ...; + # ternary = expr ? new type{} : nullptr; + # OuterTemplate{}> + # + # We check for the character following the closing brace, and + # silence the warning if it's one of those listed above, i.e. + # "{.;,)<>]:". + # + # To account for nested initializer list, we allow any number of + # closing braces up to "{;,)<". We can't simply silence the + # warning on first sight of closing brace, because that would + # cause false negatives for things that are not initializer lists. + # Silence this: But not this: + # Outer{ if (...) { + # Inner{...} if (...){ // Missing space before { + # }; } + # + # There is a false negative with this approach if people inserted + # spurious semicolons, e.g. "if (cond){};", but we will catch the + # spurious semicolon with a separate check. + (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + trailing_text = '' + if endpos > -1: + trailing_text = endline[endpos:] + for offset in xrange(endlinenum + 1, + min(endlinenum + 3, clean_lines.NumLines() - 1)): + trailing_text += clean_lines.elided[offset] + if not Match(r'^[\s}]*[{.;,)<>\]:]', trailing_text): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before {') + + # Make sure '} else {' has spaces. + if Search(r'}else', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before else') + + # You shouldn't have a space before a semicolon at the end of the line. + # There's a special case for "for" since the style guide allows space before + # the semicolon there. + if Search(r':\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement. Use {} instead.') + elif Search(r'^\s*;\s*$', line): + error( + filename, linenum, 'whitespace/semicolon', 5, + 'Line contains only semicolon. If this should be an empty statement, ' + 'use {} instead.') + elif (Search(r'\s+;\s*$', line) and not Search(r'\bfor\b', line)): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Extra space before last semicolon. If this should be an empty ' + 'statement, use {} instead.') def IsDecltype(clean_lines, linenum, column): - """Check if the token ending on (linenum, column) is decltype(). + """Check if the token ending on (linenum, column) is decltype(). Args: clean_lines: A CleansedLines instance containing the file. @@ -3411,16 +3426,16 @@ def IsDecltype(clean_lines, linenum, column): Returns: True if this token is decltype() expression, False otherwise. """ - (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) - if start_col < 0: + (text, _, start_col) = ReverseCloseExpression(clean_lines, linenum, column) + if start_col < 0: + return False + if Search(r'\bdecltype\s*$', text[0:start_col]): + return True return False - if Search(r'\bdecltype\s*$', text[0:start_col]): - return True - return False def IsTemplateParameterList(clean_lines, linenum, column): - """Check if the token ending on (linenum, column) is the end of template<>. + """Check if the token ending on (linenum, column) is the end of template<>. Args: clean_lines: A CleansedLines instance containing the file. @@ -3429,16 +3444,16 @@ def IsTemplateParameterList(clean_lines, linenum, column): Returns: True if this token is end of a template parameter list, False otherwise. """ - (_, startline, startpos) = ReverseCloseExpression( - clean_lines, linenum, column) - if (startpos > -1 and - Search(r'\btemplate\s*$', clean_lines.elided[startline][0:startpos])): - return True - return False + (_, startline, startpos) = ReverseCloseExpression(clean_lines, linenum, + column) + if (startpos > -1 and Search(r'\btemplate\s*$', + clean_lines.elided[startline][0:startpos])): + return True + return False def IsRValueType(typenames, clean_lines, nesting_state, linenum, column): - """Check if the token ending on (linenum, column) is a type. + """Check if the token ending on (linenum, column) is a type. Assumes that text to the right of the column is "&&" or a function name. @@ -3453,196 +3468,198 @@ def IsRValueType(typenames, clean_lines, nesting_state, linenum, column): Returns: True if this token is a type, False if we are not sure. """ - prefix = clean_lines.elided[linenum][0:column] - - # Get one word to the left. If we failed to do so, this is most - # likely not a type, since it's unlikely that the type name and "&&" - # would be split across multiple lines. - match = Match(r'^(.*)(\b\w+|[>*)&])\s*$', prefix) - if not match: - return False + prefix = clean_lines.elided[linenum][0:column] - # Check text following the token. If it's "&&>" or "&&," or "&&...", it's - # most likely a rvalue reference used inside a template. - suffix = clean_lines.elided[linenum][column:] - if Match(r'&&\s*(?:[>,]|\.\.\.)', suffix): - return True + # Get one word to the left. If we failed to do so, this is most + # likely not a type, since it's unlikely that the type name and "&&" + # would be split across multiple lines. + match = Match(r'^(.*)(\b\w+|[>*)&])\s*$', prefix) + if not match: + return False - # Check for known types and end of templates: - # int&& variable - # vector&& variable - # - # Because this function is called recursively, we also need to - # recognize pointer and reference types: - # int* Function() - # int& Function() - if (match.group(2) in typenames or - match.group(2) in ['char', 'char16_t', 'char32_t', 'wchar_t', 'bool', - 'short', 'int', 'long', 'signed', 'unsigned', - 'float', 'double', 'void', 'auto', '>', '*', '&']): - return True + # Check text following the token. If it's "&&>" or "&&," or "&&...", it's + # most likely a rvalue reference used inside a template. + suffix = clean_lines.elided[linenum][column:] + if Match(r'&&\s*(?:[>,]|\.\.\.)', suffix): + return True - # If we see a close parenthesis, look for decltype on the other side. - # decltype would unambiguously identify a type, anything else is - # probably a parenthesized expression and not a type. - if match.group(2) == ')': - return IsDecltype( - clean_lines, linenum, len(match.group(1)) + len(match.group(2)) - 1) - - # Check for casts and cv-qualifiers. - # match.group(1) remainder - # -------------- --------- - # const_cast< type&& - # const type&& - # type const&& - if Search(r'\b(?:const_cast\s*<|static_cast\s*<|dynamic_cast\s*<|' - r'reinterpret_cast\s*<|\w+\s)\s*$', - match.group(1)): - return True + # Check for known types and end of templates: + # int&& variable + # vector&& variable + # + # Because this function is called recursively, we also need to + # recognize pointer and reference types: + # int* Function() + # int& Function() + if (match.group(2) in typenames or match.group(2) in [ + 'char', 'char16_t', 'char32_t', 'wchar_t', 'bool', 'short', 'int', + 'long', 'signed', 'unsigned', 'float', 'double', 'void', 'auto', + '>', '*', '&' + ]): + return True - # Look for a preceding symbol that might help differentiate the context. - # These are the cases that would be ambiguous: - # match.group(1) remainder - # -------------- --------- - # Call ( expression && - # Declaration ( type&& - # sizeof ( type&& - # if ( expression && - # while ( expression && - # for ( type&& - # for( ; expression && - # statement ; type&& - # block { type&& - # constructor { expression && - start = linenum - line = match.group(1) - match_symbol = None - while start >= 0: - # We want to skip over identifiers and commas to get to a symbol. - # Commas are skipped so that we can find the opening parenthesis - # for function parameter lists. - match_symbol = Match(r'^(.*)([^\w\s,])[\w\s,]*$', line) - if match_symbol: - break - start -= 1 - line = clean_lines.elided[start] - - if not match_symbol: - # Probably the first statement in the file is an rvalue reference - return True + # If we see a close parenthesis, look for decltype on the other side. + # decltype would unambiguously identify a type, anything else is + # probably a parenthesized expression and not a type. + if match.group(2) == ')': + return IsDecltype(clean_lines, linenum, + len(match.group(1)) + len(match.group(2)) - 1) + + # Check for casts and cv-qualifiers. + # match.group(1) remainder + # -------------- --------- + # const_cast< type&& + # const type&& + # type const&& + if Search(r'\b(?:const_cast\s*<|static_cast\s*<|dynamic_cast\s*<|' + r'reinterpret_cast\s*<|\w+\s)\s*$', match.group(1)): + return True - if match_symbol.group(2) == '}': - # Found closing brace, probably an indicate of this: - # block{} type&& - return True + # Look for a preceding symbol that might help differentiate the context. + # These are the cases that would be ambiguous: + # match.group(1) remainder + # -------------- --------- + # Call ( expression && + # Declaration ( type&& + # sizeof ( type&& + # if ( expression && + # while ( expression && + # for ( type&& + # for( ; expression && + # statement ; type&& + # block { type&& + # constructor { expression && + start = linenum + line = match.group(1) + match_symbol = None + while start >= 0: + # We want to skip over identifiers and commas to get to a symbol. + # Commas are skipped so that we can find the opening parenthesis + # for function parameter lists. + match_symbol = Match(r'^(.*)([^\w\s,])[\w\s,]*$', line) + if match_symbol: + break + start -= 1 + line = clean_lines.elided[start] - if match_symbol.group(2) == ';': - # Found semicolon, probably one of these: - # for(; expression && - # statement; type&& - - # Look for the previous 'for(' in the previous lines. - before_text = match_symbol.group(1) - for i in xrange(start - 1, max(start - 6, 0), -1): - before_text = clean_lines.elided[i] + before_text - if Search(r'for\s*\([^{};]*$', before_text): - # This is the condition inside a for-loop - return False - - # Did not find a for-init-statement before this semicolon, so this - # is probably a new statement and not a condition. - return True + if not match_symbol: + # Probably the first statement in the file is an rvalue reference + return True - if match_symbol.group(2) == '{': - # Found opening brace, probably one of these: - # block{ type&& = ... ; } - # constructor{ expression && expression } + if match_symbol.group(2) == '}': + # Found closing brace, probably an indicate of this: + # block{} type&& + return True - # Look for a closing brace or a semicolon. If we see a semicolon - # first, this is probably a rvalue reference. - line = clean_lines.elided[start][0:len(match_symbol.group(1)) + 1] - end = start - depth = 1 - while True: - for ch in line: - if ch == ';': - return True - elif ch == '{': - depth += 1 - elif ch == '}': - depth -= 1 - if depth == 0: + if match_symbol.group(2) == ';': + # Found semicolon, probably one of these: + # for(; expression && + # statement; type&& + + # Look for the previous 'for(' in the previous lines. + before_text = match_symbol.group(1) + for i in xrange(start - 1, max(start - 6, 0), -1): + before_text = clean_lines.elided[i] + before_text + if Search(r'for\s*\([^{};]*$', before_text): + # This is the condition inside a for-loop return False - end += 1 - if end >= clean_lines.NumLines(): - break - line = clean_lines.elided[end] - # Incomplete program? - return False - if match_symbol.group(2) == '(': - # Opening parenthesis. Need to check what's to the left of the - # parenthesis. Look back one extra line for additional context. - before_text = match_symbol.group(1) - if linenum > 1: - before_text = clean_lines.elided[linenum - 1] + before_text - before_text = match_symbol.group(1) - - # Patterns that are likely to be types: - # [](type&& - # for (type&& - # sizeof(type&& - # operator=(type&& - # - if Search(r'(?:\]|\bfor|\bsizeof|\boperator\s*\S+\s*)\s*$', before_text): - return True - - # Patterns that are likely to be expressions: - # if (expression && - # while (expression && - # : initializer(expression && - # , initializer(expression && - # ( FunctionCall(expression && - # + FunctionCall(expression && - # + (expression && - # - # The last '+' represents operators such as '+' and '-'. - if Search(r'(?:\bif|\bwhile|[-+=%^(]*>)?\s*$', - match_symbol.group(1)) - if match_func: - # Check for constructors, which don't have return types. - if Search(r'\b(?:explicit|inline)$', match_func.group(1)): - return True - implicit_constructor = Match(r'\s*(\w+)\((?:const\s+)?(\w+)', prefix) - if (implicit_constructor and - implicit_constructor.group(1) == implicit_constructor.group(2)): + # Did not find a for-init-statement before this semicolon, so this + # is probably a new statement and not a condition. return True - return IsRValueType(typenames, clean_lines, nesting_state, linenum, - len(match_func.group(1))) - # Nothing before the function name. If this is inside a block scope, - # this is probably a function call. - return not (nesting_state.previous_stack_top and - nesting_state.previous_stack_top.IsBlockInfo()) + if match_symbol.group(2) == '{': + # Found opening brace, probably one of these: + # block{ type&& = ... ; } + # constructor{ expression && expression } + + # Look for a closing brace or a semicolon. If we see a semicolon + # first, this is probably a rvalue reference. + line = clean_lines.elided[start][0:len(match_symbol.group(1)) + 1] + end = start + depth = 1 + while True: + for ch in line: + if ch == ';': + return True + elif ch == '{': + depth += 1 + elif ch == '}': + depth -= 1 + if depth == 0: + return False + end += 1 + if end >= clean_lines.NumLines(): + break + line = clean_lines.elided[end] + # Incomplete program? + return False - if match_symbol.group(2) == '>': - # Possibly a closing bracket, check that what's on the other side - # looks like the start of a template. - return IsTemplateParameterList( - clean_lines, start, len(match_symbol.group(1))) + if match_symbol.group(2) == '(': + # Opening parenthesis. Need to check what's to the left of the + # parenthesis. Look back one extra line for additional context. + before_text = match_symbol.group(1) + if linenum > 1: + before_text = clean_lines.elided[linenum - 1] + before_text + before_text = match_symbol.group(1) + + # Patterns that are likely to be types: + # [](type&& + # for (type&& + # sizeof(type&& + # operator=(type&& + # + if Search(r'(?:\]|\bfor|\bsizeof|\boperator\s*\S+\s*)\s*$', + before_text): + return True + + # Patterns that are likely to be expressions: + # if (expression && + # while (expression && + # : initializer(expression && + # , initializer(expression && + # ( FunctionCall(expression && + # + FunctionCall(expression && + # + (expression && + # + # The last '+' represents operators such as '+' and '-'. + if Search(r'(?:\bif|\bwhile|[-+=%^(]*>)?\s*$', + match_symbol.group(1)) + if match_func: + # Check for constructors, which don't have return types. + if Search(r'\b(?:explicit|inline)$', match_func.group(1)): + return True + implicit_constructor = Match(r'\s*(\w+)\((?:const\s+)?(\w+)', + prefix) + if (implicit_constructor and implicit_constructor.group(1) == + implicit_constructor.group(2)): + return True + return IsRValueType(typenames, clean_lines, nesting_state, linenum, + len(match_func.group(1))) + + # Nothing before the function name. If this is inside a block scope, + # this is probably a function call. + return not (nesting_state.previous_stack_top and + nesting_state.previous_stack_top.IsBlockInfo()) + + if match_symbol.group(2) == '>': + # Possibly a closing bracket, check that what's on the other side + # looks like the start of a template. + return IsTemplateParameterList(clean_lines, start, + len(match_symbol.group(1))) + + # Some other symbol, usually something like "a=b&&c". This is most + # likely not a type. + return False def IsDeletedOrDefault(clean_lines, linenum): - """Check if current constructor or operator is deleted or default. + """Check if current constructor or operator is deleted or default. Args: clean_lines: A CleansedLines instance containing the file. @@ -3650,18 +3667,18 @@ def IsDeletedOrDefault(clean_lines, linenum): Returns: True if this is a deleted or default constructor. """ - open_paren = clean_lines.elided[linenum].find('(') - if open_paren < 0: - return False - (close_line, _, close_paren) = CloseExpression( - clean_lines, linenum, open_paren) - if close_paren < 0: - return False - return Match(r'\s*=\s*(?:delete|default)\b', close_line[close_paren:]) + open_paren = clean_lines.elided[linenum].find('(') + if open_paren < 0: + return False + (close_line, _, close_paren) = CloseExpression(clean_lines, linenum, + open_paren) + if close_paren < 0: + return False + return Match(r'\s*=\s*(?:delete|default)\b', close_line[close_paren:]) def IsRValueAllowed(clean_lines, linenum, typenames): - """Check if RValue reference is allowed on a particular line. + """Check if RValue reference is allowed on a particular line. Args: clean_lines: A CleansedLines instance containing the file. @@ -3670,56 +3687,57 @@ def IsRValueAllowed(clean_lines, linenum, typenames): Returns: True if line is within the region where RValue references are allowed. """ - # Allow region marked by PUSH/POP macros - for i in xrange(linenum, 0, -1): - line = clean_lines.elided[i] - if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): - if not line.endswith('PUSH'): - return False - for j in xrange(linenum, clean_lines.NumLines(), 1): - line = clean_lines.elided[j] + # Allow region marked by PUSH/POP macros + for i in xrange(linenum, 0, -1): + line = clean_lines.elided[i] if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): - return line.endswith('POP') - - # Allow operator= - line = clean_lines.elided[linenum] - if Search(r'\boperator\s*=\s*\(', line): - return IsDeletedOrDefault(clean_lines, linenum) - - # Allow constructors - match = Match(r'\s*(?:[\w<>]+::)*([\w<>]+)\s*::\s*([\w<>]+)\s*\(', line) - if match and match.group(1) == match.group(2): - return IsDeletedOrDefault(clean_lines, linenum) - if Search(r'\b(?:explicit|inline)\s+[\w<>]+\s*\(', line): - return IsDeletedOrDefault(clean_lines, linenum) - - if Match(r'\s*[\w<>]+\s*\(', line): - previous_line = 'ReturnType' - if linenum > 0: - previous_line = clean_lines.elided[linenum - 1] - if Match(r'^\s*$', previous_line) or Search(r'[{}:;]\s*$', previous_line): - return IsDeletedOrDefault(clean_lines, linenum) + if not line.endswith('PUSH'): + return False + for j in xrange(linenum, clean_lines.NumLines(), 1): + line = clean_lines.elided[j] + if Match(r'GOOGLE_ALLOW_RVALUE_REFERENCES_(?:PUSH|POP)', line): + return line.endswith('POP') - # Reject types not mentioned in template-argument-list - while line: - match = Match(r'^.*?(\w+)\s*&&(.*)$', line) - if not match: - break - if match.group(1) not in typenames: - return False - line = match.group(2) + # Allow operator= + line = clean_lines.elided[linenum] + if Search(r'\boperator\s*=\s*\(', line): + return IsDeletedOrDefault(clean_lines, linenum) + + # Allow constructors + match = Match(r'\s*(?:[\w<>]+::)*([\w<>]+)\s*::\s*([\w<>]+)\s*\(', line) + if match and match.group(1) == match.group(2): + return IsDeletedOrDefault(clean_lines, linenum) + if Search(r'\b(?:explicit|inline)\s+[\w<>]+\s*\(', line): + return IsDeletedOrDefault(clean_lines, linenum) + + if Match(r'\s*[\w<>]+\s*\(', line): + previous_line = 'ReturnType' + if linenum > 0: + previous_line = clean_lines.elided[linenum - 1] + if Match(r'^\s*$', previous_line) or Search(r'[{}:;]\s*$', + previous_line): + return IsDeletedOrDefault(clean_lines, linenum) + + # Reject types not mentioned in template-argument-list + while line: + match = Match(r'^.*?(\w+)\s*&&(.*)$', line) + if not match: + break + if match.group(1) not in typenames: + return False + line = match.group(2) - # All RValue types that were in template-argument-list should have - # been removed by now. Those were allowed, assuming that they will - # be forwarded. - # - # If there are no remaining RValue types left (i.e. types that were - # not found in template-argument-list), flag those as not allowed. - return line.find('&&') < 0 + # All RValue types that were in template-argument-list should have + # been removed by now. Those were allowed, assuming that they will + # be forwarded. + # + # If there are no remaining RValue types left (i.e. types that were + # not found in template-argument-list), flag those as not allowed. + return line.find('&&') < 0 def GetTemplateArgs(clean_lines, linenum): - """Find list of template arguments associated with this function declaration. + """Find list of template arguments associated with this function declaration. Args: clean_lines: A CleansedLines instance containing the file. @@ -3729,61 +3747,63 @@ def GetTemplateArgs(clean_lines, linenum): Set of type names, or empty set if this does not appear to have any template parameters. """ - # Find start of function - func_line = linenum - while func_line > 0: - line = clean_lines.elided[func_line] - if Match(r'^\s*$', line): - return set() - if line.find('(') >= 0: - break - func_line -= 1 - if func_line == 0: - return set() - - # Collapse template-argument-list into a single string - argument_list = '' - match = Match(r'^(\s*template\s*)<', clean_lines.elided[func_line]) - if match: - # template-argument-list on the same line as function name - start_col = len(match.group(1)) - _, end_line, end_col = CloseExpression(clean_lines, func_line, start_col) - if end_col > -1 and end_line == func_line: - start_col += 1 # Skip the opening bracket - argument_list = clean_lines.elided[func_line][start_col:end_col] - - elif func_line > 1: - # template-argument-list one line before function name - match = Match(r'^(.*)>\s*$', clean_lines.elided[func_line - 1]) + # Find start of function + func_line = linenum + while func_line > 0: + line = clean_lines.elided[func_line] + if Match(r'^\s*$', line): + return set() + if line.find('(') >= 0: + break + func_line -= 1 + if func_line == 0: + return set() + + # Collapse template-argument-list into a single string + argument_list = '' + match = Match(r'^(\s*template\s*)<', clean_lines.elided[func_line]) if match: - end_col = len(match.group(1)) - _, start_line, start_col = ReverseCloseExpression( - clean_lines, func_line - 1, end_col) - if start_col > -1: - start_col += 1 # Skip the opening bracket - while start_line < func_line - 1: - argument_list += clean_lines.elided[start_line][start_col:] - start_col = 0 - start_line += 1 - argument_list += clean_lines.elided[func_line - 1][start_col:end_col] - - if not argument_list: - return set() - - # Extract type names - typenames = set() - while True: - match = Match(r'^[,\s]*(?:typename|class)(?:\.\.\.)?\s+(\w+)(.*)$', - argument_list) - if not match: - break - typenames.add(match.group(1)) - argument_list = match.group(2) - return typenames + # template-argument-list on the same line as function name + start_col = len(match.group(1)) + _, end_line, end_col = CloseExpression(clean_lines, func_line, + start_col) + if end_col > -1 and end_line == func_line: + start_col += 1 # Skip the opening bracket + argument_list = clean_lines.elided[func_line][start_col:end_col] + + elif func_line > 1: + # template-argument-list one line before function name + match = Match(r'^(.*)>\s*$', clean_lines.elided[func_line - 1]) + if match: + end_col = len(match.group(1)) + _, start_line, start_col = ReverseCloseExpression( + clean_lines, func_line - 1, end_col) + if start_col > -1: + start_col += 1 # Skip the opening bracket + while start_line < func_line - 1: + argument_list += clean_lines.elided[start_line][start_col:] + start_col = 0 + start_line += 1 + argument_list += clean_lines.elided[func_line - 1][start_col: + end_col] + + if not argument_list: + return set() + + # Extract type names + typenames = set() + while True: + match = Match(r'^[,\s]*(?:typename|class)(?:\.\.\.)?\s+(\w+)(.*)$', + argument_list) + if not match: + break + typenames.add(match.group(1)) + argument_list = match.group(2) + return typenames def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error): - """Check for rvalue references. + """Check for rvalue references. Args: filename: The name of the current file. @@ -3793,33 +3813,34 @@ def CheckRValueReference(filename, clean_lines, linenum, nesting_state, error): the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - # Find lines missing spaces around &&. - # TODO(unknown): currently we don't check for rvalue references - # with spaces surrounding the && to avoid false positives with - # boolean expressions. - line = clean_lines.elided[linenum] - match = Match(r'^(.*\S)&&', line) - if not match: - match = Match(r'(.*)&&\S', line) - if (not match) or '(&&)' in line or Search(r'\boperator\s*$', match.group(1)): - return - - # Either poorly formed && or an rvalue reference, check the context - # to get a more accurate error message. Mostly we want to determine - # if what's to the left of "&&" is a type or not. - typenames = GetTemplateArgs(clean_lines, linenum) - and_pos = len(match.group(1)) - if IsRValueType(typenames, clean_lines, nesting_state, linenum, and_pos): - if not IsRValueAllowed(clean_lines, linenum, typenames): - error(filename, linenum, 'build/c++11', 3, - 'RValue references are an unapproved C++ feature.') - else: - error(filename, linenum, 'whitespace/operators', 3, - 'Missing spaces around &&') + # Find lines missing spaces around &&. + # TODO(unknown): currently we don't check for rvalue references + # with spaces surrounding the && to avoid false positives with + # boolean expressions. + line = clean_lines.elided[linenum] + match = Match(r'^(.*\S)&&', line) + if not match: + match = Match(r'(.*)&&\S', line) + if (not match) or '(&&)' in line or Search(r'\boperator\s*$', + match.group(1)): + return + + # Either poorly formed && or an rvalue reference, check the context + # to get a more accurate error message. Mostly we want to determine + # if what's to the left of "&&" is a type or not. + typenames = GetTemplateArgs(clean_lines, linenum) + and_pos = len(match.group(1)) + if IsRValueType(typenames, clean_lines, nesting_state, linenum, and_pos): + if not IsRValueAllowed(clean_lines, linenum, typenames): + error(filename, linenum, 'build/c++11', 3, + 'RValue references are an unapproved C++ feature.') + else: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around &&') def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): - """Checks for additional blank line issues related to sections. + """Checks for additional blank line issues related to sections. Currently the only thing checked here is blank line before protected/private. @@ -3830,51 +3851,53 @@ def CheckSectionSpacing(filename, clean_lines, class_info, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Skip checks if the class is small, where small means 25 lines or less. - # 25 lines seems like a good cutoff since that's the usual height of - # terminals, and any class that can't fit in one screen can't really - # be considered "small". - # - # Also skip checks if we are on the first line. This accounts for - # classes that look like - # class Foo { public: ... }; - # - # If we didn't find the end of the class, last_line would be zero, - # and the check will be skipped by the first condition. - if (class_info.last_line - class_info.starting_linenum <= 24 or - linenum <= class_info.starting_linenum): - return - - matched = Match(r'\s*(public|protected|private):', clean_lines.lines[linenum]) - if matched: - # Issue warning if the line before public/protected/private was - # not a blank line, but don't do this if the previous line contains - # "class" or "struct". This can happen two ways: - # - We are at the beginning of the class. - # - We are forward-declaring an inner class that is semantically - # private, but needed to be public for implementation reasons. - # Also ignores cases where the previous line ends with a backslash as can be - # common when defining classes in C macros. - prev_line = clean_lines.lines[linenum - 1] - if (not IsBlankLine(prev_line) and - not Search(r'\b(class|struct)\b', prev_line) and - not Search(r'\\$', prev_line)): - # Try a bit harder to find the beginning of the class. This is to - # account for multi-line base-specifier lists, e.g.: - # class Derived - # : public Base { - end_class_head = class_info.starting_linenum - for i in range(class_info.starting_linenum, linenum): - if Search(r'\{\s*$', clean_lines.lines[i]): - end_class_head = i - break - if end_class_head < linenum - 1: - error(filename, linenum, 'whitespace/blank_line', 3, - '"%s:" should be preceded by a blank line' % matched.group(1)) + # Skip checks if the class is small, where small means 25 lines or less. + # 25 lines seems like a good cutoff since that's the usual height of + # terminals, and any class that can't fit in one screen can't really + # be considered "small". + # + # Also skip checks if we are on the first line. This accounts for + # classes that look like + # class Foo { public: ... }; + # + # If we didn't find the end of the class, last_line would be zero, + # and the check will be skipped by the first condition. + if (class_info.last_line - class_info.starting_linenum <= 24 or + linenum <= class_info.starting_linenum): + return + + matched = Match(r'\s*(public|protected|private):', + clean_lines.lines[linenum]) + if matched: + # Issue warning if the line before public/protected/private was + # not a blank line, but don't do this if the previous line contains + # "class" or "struct". This can happen two ways: + # - We are at the beginning of the class. + # - We are forward-declaring an inner class that is semantically + # private, but needed to be public for implementation reasons. + # Also ignores cases where the previous line ends with a backslash as can be + # common when defining classes in C macros. + prev_line = clean_lines.lines[linenum - 1] + if (not IsBlankLine(prev_line) and + not Search(r'\b(class|struct)\b', prev_line) and + not Search(r'\\$', prev_line)): + # Try a bit harder to find the beginning of the class. This is to + # account for multi-line base-specifier lists, e.g.: + # class Derived + # : public Base { + end_class_head = class_info.starting_linenum + for i in range(class_info.starting_linenum, linenum): + if Search(r'\{\s*$', clean_lines.lines[i]): + end_class_head = i + break + if end_class_head < linenum - 1: + error(filename, linenum, 'whitespace/blank_line', 3, + '"%s:" should be preceded by a blank line' % + matched.group(1)) def GetPreviousNonBlankLine(clean_lines, linenum): - """Return the most recent non-blank line and its line number. + """Return the most recent non-blank line and its line number. Args: clean_lines: A CleansedLines instance containing the file contents. @@ -3887,17 +3910,17 @@ def GetPreviousNonBlankLine(clean_lines, linenum): if this is the first non-blank line. """ - prevlinenum = linenum - 1 - while prevlinenum >= 0: - prevline = clean_lines.elided[prevlinenum] - if not IsBlankLine(prevline): # if not a blank line... - return (prevline, prevlinenum) - prevlinenum -= 1 - return ('', -1) + prevlinenum = linenum - 1 + while prevlinenum >= 0: + prevline = clean_lines.elided[prevlinenum] + if not IsBlankLine(prevline): # if not a blank line... + return (prevline, prevlinenum) + prevlinenum -= 1 + return ('', -1) def CheckBraces(filename, clean_lines, linenum, error): - """Looks for misplaced braces (e.g. at the end of line). + """Looks for misplaced braces (e.g. at the end of line). Args: filename: The name of the current file. @@ -3906,114 +3929,123 @@ def CheckBraces(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] # get rid of comments and strings - - if Match(r'\s*{\s*$', line): - # We allow an open brace to start a line in the case where someone is using - # braces in a block to explicitly create a new scope, which is commonly used - # to control the lifetime of stack-allocated variables. Braces are also - # used for brace initializers inside function calls. We don't detect this - # perfectly: we just don't complain if the last non-whitespace character on - # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the - # previous line starts a preprocessor block. - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if (not Search(r'[,;:}{(]\s*$', prevline) and - not Match(r'\s*#', prevline)): - error(filename, linenum, 'whitespace/braces', 4, - '{ should almost always be at the end of the previous line') - - # An else clause should be on the same line as the preceding closing brace. - if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if Match(r'\s*}\s*$', prevline): - error(filename, linenum, 'whitespace/newline', 4, - 'An else should appear on the same line as the preceding }') - - # If braces come on one side of an else, they should be on both. - # However, we have to worry about "else if" that spans multiple lines! - if Search(r'else if\s*\(', line): # could be multi-line if - brace_on_left = bool(Search(r'}\s*else if\s*\(', line)) - # find the ( after the if - pos = line.find('else if') - pos = line.find('(', pos) - if pos > 0: - (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) - brace_on_right = endline[endpos:].find('{') != -1 - if brace_on_left != brace_on_right: # must be brace after if + line = clean_lines.elided[linenum] # get rid of comments and strings + + if Match(r'\s*{\s*$', line): + # We allow an open brace to start a line in the case where someone is using + # braces in a block to explicitly create a new scope, which is commonly used + # to control the lifetime of stack-allocated variables. Braces are also + # used for brace initializers inside function calls. We don't detect this + # perfectly: we just don't complain if the last non-whitespace character on + # the previous non-blank line is ',', ';', ':', '(', '{', or '}', or if the + # previous line starts a preprocessor block. + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if (not Search(r'[,;:}{(]\s*$', prevline) and + not Match(r'\s*#', prevline)): + error(filename, linenum, 'whitespace/braces', 4, + '{ should almost always be at the end of the previous line') + + # An else clause should be on the same line as the preceding closing brace. + if Match(r'\s*else\b\s*(?:if\b|\{|$)', line): + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if Match(r'\s*}\s*$', prevline): + error(filename, linenum, 'whitespace/newline', 4, + 'An else should appear on the same line as the preceding }') + + # If braces come on one side of an else, they should be on both. + # However, we have to worry about "else if" that spans multiple lines! + if Search(r'else if\s*\(', line): # could be multi-line if + brace_on_left = bool(Search(r'}\s*else if\s*\(', line)) + # find the ( after the if + pos = line.find('else if') + pos = line.find('(', pos) + if pos > 0: + (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) + brace_on_right = endline[endpos:].find('{') != -1 + if brace_on_left != brace_on_right: # must be brace after if + error( + filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both' + ) + elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): error(filename, linenum, 'readability/braces', 5, 'If an else has a brace on one side, it should have it on both') - elif Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): - error(filename, linenum, 'readability/braces', 5, - 'If an else has a brace on one side, it should have it on both') - - # Likewise, an else should never have the else clause on the same line - if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): - error(filename, linenum, 'whitespace/newline', 4, - 'Else clause should never be on same line as else (use 2 lines)') - - # In the same way, a do/while should never be on one line - if Match(r'\s*do [^\s{]', line): - error(filename, linenum, 'whitespace/newline', 4, - 'do/while clauses should not be on a single line') - - # Check single-line if/else bodies. The style guide says 'curly braces are not - # required for single-line statements'. We additionally allow multi-line, - # single statements, but we reject anything with more than one semicolon in - # it. This means that the first semicolon after the if should be at the end of - # its line, and the line after that should have an indent level equal to or - # lower than the if. We also check for ambiguous if/else nesting without - # braces. - if_else_match = Search(r'\b(if\s*\(|else\b)', line) - if if_else_match and not Match(r'\s*#', line): - if_indent = GetIndentLevel(line) - endline, endlinenum, endpos = line, linenum, if_else_match.end() - if_match = Search(r'\bif\s*\(', line) - if if_match: - # This could be a multiline if condition, so find the end first. - pos = if_match.end() - 1 - (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, pos) - # Check for an opening brace, either directly after the if or on the next - # line. If found, this isn't a single-statement conditional. - if (not Match(r'\s*{', endline[endpos:]) - and not (Match(r'\s*$', endline[endpos:]) - and endlinenum < (len(clean_lines.elided) - 1) - and Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): - while (endlinenum < len(clean_lines.elided) - and ';' not in clean_lines.elided[endlinenum][endpos:]): - endlinenum += 1 - endpos = 0 - if endlinenum < len(clean_lines.elided): - endline = clean_lines.elided[endlinenum] - # We allow a mix of whitespace and closing braces (e.g. for one-liner - # methods) and a single \ after the semicolon (for macros) - endpos = endline.find(';') - if not Match(r';[\s}]*(\\?)$', endline[endpos:]): - # Semicolon isn't the last character, there's something trailing. - # Output a warning if the semicolon is not contained inside - # a lambda expression. - if not Match(r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', - endline): - error(filename, linenum, 'readability/braces', 4, - 'If/else bodies with multiple statements require braces') - elif endlinenum < len(clean_lines.elided) - 1: - # Make sure the next line is dedented - next_line = clean_lines.elided[endlinenum + 1] - next_indent = GetIndentLevel(next_line) - # With ambiguous nested if statements, this will error out on the - # if that *doesn't* match the else, regardless of whether it's the - # inner one or outer one. - if (if_match and Match(r'\s*else\b', next_line) - and next_indent != if_indent): - error(filename, linenum, 'readability/braces', 4, - 'Else clause should be indented at the same level as if. ' - 'Ambiguous nested if/else chains require braces.') - elif next_indent > if_indent: - error(filename, linenum, 'readability/braces', 4, - 'If/else bodies with multiple statements require braces') + + # Likewise, an else should never have the else clause on the same line + if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): + error(filename, linenum, 'whitespace/newline', 4, + 'Else clause should never be on same line as else (use 2 lines)') + + # In the same way, a do/while should never be on one line + if Match(r'\s*do [^\s{]', line): + error(filename, linenum, 'whitespace/newline', 4, + 'do/while clauses should not be on a single line') + + # Check single-line if/else bodies. The style guide says 'curly braces are not + # required for single-line statements'. We additionally allow multi-line, + # single statements, but we reject anything with more than one semicolon in + # it. This means that the first semicolon after the if should be at the end of + # its line, and the line after that should have an indent level equal to or + # lower than the if. We also check for ambiguous if/else nesting without + # braces. + if_else_match = Search(r'\b(if\s*\(|else\b)', line) + if if_else_match and not Match(r'\s*#', line): + if_indent = GetIndentLevel(line) + endline, endlinenum, endpos = line, linenum, if_else_match.end() + if_match = Search(r'\bif\s*\(', line) + if if_match: + # This could be a multiline if condition, so find the end first. + pos = if_match.end() - 1 + (endline, endlinenum, endpos) = CloseExpression(clean_lines, + linenum, pos) + # Check for an opening brace, either directly after the if or on the next + # line. If found, this isn't a single-statement conditional. + if (not Match(r'\s*{', endline[endpos:]) and + not (Match(r'\s*$', endline[endpos:]) and endlinenum < + (len(clean_lines.elided) - 1) and + Match(r'\s*{', clean_lines.elided[endlinenum + 1]))): + while (endlinenum < len(clean_lines.elided) and + ';' not in clean_lines.elided[endlinenum][endpos:]): + endlinenum += 1 + endpos = 0 + if endlinenum < len(clean_lines.elided): + endline = clean_lines.elided[endlinenum] + # We allow a mix of whitespace and closing braces (e.g. for one-liner + # methods) and a single \ after the semicolon (for macros) + endpos = endline.find(';') + if not Match(r';[\s}]*(\\?)$', endline[endpos:]): + # Semicolon isn't the last character, there's something trailing. + # Output a warning if the semicolon is not contained inside + # a lambda expression. + if not Match( + r'^[^{};]*\[[^\[\]]*\][^{}]*\{[^{}]*\}\s*\)*[;,]\s*$', + endline): + error( + filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces' + ) + elif endlinenum < len(clean_lines.elided) - 1: + # Make sure the next line is dedented + next_line = clean_lines.elided[endlinenum + 1] + next_indent = GetIndentLevel(next_line) + # With ambiguous nested if statements, this will error out on the + # if that *doesn't* match the else, regardless of whether it's the + # inner one or outer one. + if (if_match and Match(r'\s*else\b', next_line) and + next_indent != if_indent): + error( + filename, linenum, 'readability/braces', 4, + 'Else clause should be indented at the same level as if. ' + 'Ambiguous nested if/else chains require braces.') + elif next_indent > if_indent: + error( + filename, linenum, 'readability/braces', 4, + 'If/else bodies with multiple statements require braces' + ) def CheckTrailingSemicolon(filename, clean_lines, linenum, error): - """Looks for redundant trailing semicolon. + """Looks for redundant trailing semicolon. Args: filename: The name of the current file. @@ -4022,135 +4054,133 @@ def CheckTrailingSemicolon(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Block bodies should not be followed by a semicolon. Due to C++11 - # brace initialization, there are more places where semicolons are - # required than not, so we use a whitelist approach to check these - # rather than a blacklist. These are the places where "};" should - # be replaced by just "}": - # 1. Some flavor of block following closing parenthesis: - # for (;;) {}; - # while (...) {}; - # switch (...) {}; - # Function(...) {}; - # if (...) {}; - # if (...) else if (...) {}; - # - # 2. else block: - # if (...) else {}; - # - # 3. const member function: - # Function(...) const {}; - # - # 4. Block following some statement: - # x = 42; - # {}; - # - # 5. Block at the beginning of a function: - # Function(...) { - # {}; - # } - # - # Note that naively checking for the preceding "{" will also match - # braces inside multi-dimensional arrays, but this is fine since - # that expression will not contain semicolons. - # - # 6. Block following another block: - # while (true) {} - # {}; - # - # 7. End of namespaces: - # namespace {}; - # - # These semicolons seems far more common than other kinds of - # redundant semicolons, possibly due to people converting classes - # to namespaces. For now we do not warn for this case. - # - # Try matching case 1 first. - match = Match(r'^(.*\)\s*)\{', line) - if match: - # Matched closing parenthesis (case 1). Check the token before the - # matching opening parenthesis, and don't warn if it looks like a - # macro. This avoids these false positives: - # - macro that defines a base class - # - multi-line macro that defines a base class - # - macro that defines the whole class-head + line = clean_lines.elided[linenum] + + # Block bodies should not be followed by a semicolon. Due to C++11 + # brace initialization, there are more places where semicolons are + # required than not, so we use a whitelist approach to check these + # rather than a blacklist. These are the places where "};" should + # be replaced by just "}": + # 1. Some flavor of block following closing parenthesis: + # for (;;) {}; + # while (...) {}; + # switch (...) {}; + # Function(...) {}; + # if (...) {}; + # if (...) else if (...) {}; # - # But we still issue warnings for macros that we know are safe to - # warn, specifically: - # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P - # - TYPED_TEST - # - INTERFACE_DEF - # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # 2. else block: + # if (...) else {}; # - # We implement a whitelist of safe macros instead of a blacklist of - # unsafe macros, even though the latter appears less frequently in - # google code and would have been easier to implement. This is because - # the downside for getting the whitelist wrong means some extra - # semicolons, while the downside for getting the blacklist wrong - # would result in compile errors. + # 3. const member function: + # Function(...) const {}; # - # In addition to macros, we also don't want to warn on - # - Compound literals - # - Lambdas - # - alignas specifier with anonymous structs: - closing_brace_pos = match.group(1).rfind(')') - opening_parenthesis = ReverseCloseExpression( - clean_lines, linenum, closing_brace_pos) - if opening_parenthesis[2] > -1: - line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] - macro = Search(r'\b([A-Z_]+)\s*$', line_prefix) - func = Match(r'^(.*\])\s*$', line_prefix) - if ((macro and - macro.group(1) not in ( - 'TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', - 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', - 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or - (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or - Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or - Search(r'\s+=\s*$', line_prefix)): - match = None - if (match and - opening_parenthesis[1] > 1 and - Search(r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): - # Multi-line lambda-expression - match = None - - else: - # Try matching cases 2-3. - match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) - if not match: - # Try matching cases 4-6. These are always matched on separate lines. - # - # Note that we can't simply concatenate the previous line to the - # current line and do a single match, otherwise we may output - # duplicate warnings for the blank line case: - # if (cond) { - # // blank line - # } - prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] - if prevline and Search(r'[;{}]\s*$', prevline): - match = Match(r'^(\s*)\{', line) - - # Check matching closing brace - if match: - (endline, endlinenum, endpos) = CloseExpression( - clean_lines, linenum, len(match.group(1))) - if endpos > -1 and Match(r'^\s*;', endline[endpos:]): - # Current {} pair is eligible for semicolon check, and we have found - # the redundant semicolon, output warning here. - # - # Note: because we are scanning forward for opening braces, and - # outputting warnings for the matching closing brace, if there are - # nested blocks with trailing semicolons, we will get the error - # messages in reversed order. - error(filename, endlinenum, 'readability/braces', 4, - "You don't need a ; after a }") + # 4. Block following some statement: + # x = 42; + # {}; + # + # 5. Block at the beginning of a function: + # Function(...) { + # {}; + # } + # + # Note that naively checking for the preceding "{" will also match + # braces inside multi-dimensional arrays, but this is fine since + # that expression will not contain semicolons. + # + # 6. Block following another block: + # while (true) {} + # {}; + # + # 7. End of namespaces: + # namespace {}; + # + # These semicolons seems far more common than other kinds of + # redundant semicolons, possibly due to people converting classes + # to namespaces. For now we do not warn for this case. + # + # Try matching case 1 first. + match = Match(r'^(.*\)\s*)\{', line) + if match: + # Matched closing parenthesis (case 1). Check the token before the + # matching opening parenthesis, and don't warn if it looks like a + # macro. This avoids these false positives: + # - macro that defines a base class + # - multi-line macro that defines a base class + # - macro that defines the whole class-head + # + # But we still issue warnings for macros that we know are safe to + # warn, specifically: + # - TEST, TEST_F, TEST_P, MATCHER, MATCHER_P + # - TYPED_TEST + # - INTERFACE_DEF + # - EXCLUSIVE_LOCKS_REQUIRED, SHARED_LOCKS_REQUIRED, LOCKS_EXCLUDED: + # + # We implement a whitelist of safe macros instead of a blacklist of + # unsafe macros, even though the latter appears less frequently in + # google code and would have been easier to implement. This is because + # the downside for getting the whitelist wrong means some extra + # semicolons, while the downside for getting the blacklist wrong + # would result in compile errors. + # + # In addition to macros, we also don't want to warn on + # - Compound literals + # - Lambdas + # - alignas specifier with anonymous structs: + closing_brace_pos = match.group(1).rfind(')') + opening_parenthesis = ReverseCloseExpression(clean_lines, linenum, + closing_brace_pos) + if opening_parenthesis[2] > -1: + line_prefix = opening_parenthesis[0][0:opening_parenthesis[2]] + macro = Search(r'\b([A-Z_]+)\s*$', line_prefix) + func = Match(r'^(.*\])\s*$', line_prefix) + if ((macro and macro.group(1) not in + ('TEST', 'TEST_F', 'MATCHER', 'MATCHER_P', 'TYPED_TEST', + 'EXCLUSIVE_LOCKS_REQUIRED', 'SHARED_LOCKS_REQUIRED', + 'LOCKS_EXCLUDED', 'INTERFACE_DEF')) or + (func and not Search(r'\boperator\s*\[\s*\]', func.group(1))) or + Search(r'\b(?:struct|union)\s+alignas\s*$', line_prefix) or + Search(r'\s+=\s*$', line_prefix)): + match = None + if (match and opening_parenthesis[1] > 1 and Search( + r'\]\s*$', clean_lines.elided[opening_parenthesis[1] - 1])): + # Multi-line lambda-expression + match = None + + else: + # Try matching cases 2-3. + match = Match(r'^(.*(?:else|\)\s*const)\s*)\{', line) + if not match: + # Try matching cases 4-6. These are always matched on separate lines. + # + # Note that we can't simply concatenate the previous line to the + # current line and do a single match, otherwise we may output + # duplicate warnings for the blank line case: + # if (cond) { + # // blank line + # } + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if prevline and Search(r'[;{}]\s*$', prevline): + match = Match(r'^(\s*)\{', line) + + # Check matching closing brace + if match: + (endline, endlinenum, endpos) = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if endpos > -1 and Match(r'^\s*;', endline[endpos:]): + # Current {} pair is eligible for semicolon check, and we have found + # the redundant semicolon, output warning here. + # + # Note: because we are scanning forward for opening braces, and + # outputting warnings for the matching closing brace, if there are + # nested blocks with trailing semicolons, we will get the error + # messages in reversed order. + error(filename, endlinenum, 'readability/braces', 4, + "You don't need a ; after a }") def CheckEmptyBlockBody(filename, clean_lines, linenum, error): - """Look for empty loop/conditional body with only a single semicolon. + """Look for empty loop/conditional body with only a single semicolon. Args: filename: The name of the current file. @@ -4159,33 +4189,34 @@ def CheckEmptyBlockBody(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - # Search for loop keywords at the beginning of the line. Because only - # whitespaces are allowed before the keywords, this will also ignore most - # do-while-loops, since those lines should start with closing brace. - # - # We also check "if" blocks here, since an empty conditional block - # is likely an error. - line = clean_lines.elided[linenum] - matched = Match(r'\s*(for|while|if)\s*\(', line) - if matched: - # Find the end of the conditional expression - (end_line, end_linenum, end_pos) = CloseExpression( - clean_lines, linenum, line.find('(')) - - # Output warning if what follows the condition expression is a semicolon. - # No warning for all other cases, including whitespace or newline, since we - # have a separate check for semicolons preceded by whitespace. - if end_pos >= 0 and Match(r';', end_line[end_pos:]): - if matched.group(1) == 'if': - error(filename, end_linenum, 'whitespace/empty_conditional_body', 5, - 'Empty conditional bodies should use {}') - else: - error(filename, end_linenum, 'whitespace/empty_loop_body', 5, - 'Empty loop bodies should use {} or continue') + # Search for loop keywords at the beginning of the line. Because only + # whitespaces are allowed before the keywords, this will also ignore most + # do-while-loops, since those lines should start with closing brace. + # + # We also check "if" blocks here, since an empty conditional block + # is likely an error. + line = clean_lines.elided[linenum] + matched = Match(r'\s*(for|while|if)\s*\(', line) + if matched: + # Find the end of the conditional expression + (end_line, end_linenum, end_pos) = CloseExpression(clean_lines, linenum, + line.find('(')) + + # Output warning if what follows the condition expression is a semicolon. + # No warning for all other cases, including whitespace or newline, since we + # have a separate check for semicolons preceded by whitespace. + if end_pos >= 0 and Match(r';', end_line[end_pos:]): + if matched.group(1) == 'if': + error(filename, end_linenum, + 'whitespace/empty_conditional_body', 5, + 'Empty conditional bodies should use {}') + else: + error(filename, end_linenum, 'whitespace/empty_loop_body', 5, + 'Empty loop bodies should use {} or continue') def FindCheckMacro(line): - """Find a replaceable CHECK-like macro. + """Find a replaceable CHECK-like macro. Args: line: line to search on. @@ -4193,22 +4224,22 @@ def FindCheckMacro(line): (macro name, start position), or (None, -1) if no replaceable macro is found. """ - for macro in _CHECK_MACROS: - i = line.find(macro) - if i >= 0: - # Find opening parenthesis. Do a regular expression match here - # to make sure that we are matching the expected CHECK macro, as - # opposed to some other macro that happens to contain the CHECK - # substring. - matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) - if not matched: - continue - return (macro, len(matched.group(1))) - return (None, -1) + for macro in _CHECK_MACROS: + i = line.find(macro) + if i >= 0: + # Find opening parenthesis. Do a regular expression match here + # to make sure that we are matching the expected CHECK macro, as + # opposed to some other macro that happens to contain the CHECK + # substring. + matched = Match(r'^(.*\b' + macro + r'\s*)\(', line) + if not matched: + continue + return (macro, len(matched.group(1))) + return (None, -1) def CheckCheck(filename, clean_lines, linenum, error): - """Checks the use of CHECK and EXPECT macros. + """Checks the use of CHECK and EXPECT macros. Args: filename: The name of the current file. @@ -4217,116 +4248,116 @@ def CheckCheck(filename, clean_lines, linenum, error): error: The function to call with any errors found. """ - # Decide the set of replacement macros that should be suggested - lines = clean_lines.elided - (check_macro, start_pos) = FindCheckMacro(lines[linenum]) - if not check_macro: - return - - # Find end of the boolean expression by matching parentheses - (last_line, end_line, end_pos) = CloseExpression( - clean_lines, linenum, start_pos) - if end_pos < 0: - return - - # If the check macro is followed by something other than a - # semicolon, assume users will log their own custom error messages - # and don't suggest any replacements. - if not Match(r'\s*;', last_line[end_pos:]): - return - - if linenum == end_line: - expression = lines[linenum][start_pos + 1:end_pos - 1] - else: - expression = lines[linenum][start_pos + 1:] - for i in xrange(linenum + 1, end_line): - expression += lines[i] - expression += last_line[0:end_pos - 1] - - # Parse expression so that we can take parentheses into account. - # This avoids false positives for inputs like "CHECK((a < 4) == b)", - # which is not replaceable by CHECK_LE. - lhs = '' - rhs = '' - operator = None - while expression: - matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' - r'==|!=|>=|>|<=|<|\()(.*)$', expression) - if matched: - token = matched.group(1) - if token == '(': - # Parenthesized operand - expression = matched.group(2) - (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) - if end < 0: - return # Unmatched parenthesis - lhs += '(' + expression[0:end] - expression = expression[end:] - elif token in ('&&', '||'): - # Logical and/or operators. This means the expression - # contains more than one term, for example: - # CHECK(42 < a && a < b); - # - # These are not replaceable with CHECK_LE, so bail out early. + # Decide the set of replacement macros that should be suggested + lines = clean_lines.elided + (check_macro, start_pos) = FindCheckMacro(lines[linenum]) + if not check_macro: return - elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): - # Non-relational operator - lhs += token - expression = matched.group(2) - else: - # Relational operator - operator = token - rhs = matched.group(2) - break + + # Find end of the boolean expression by matching parentheses + (last_line, end_line, end_pos) = CloseExpression(clean_lines, linenum, + start_pos) + if end_pos < 0: + return + + # If the check macro is followed by something other than a + # semicolon, assume users will log their own custom error messages + # and don't suggest any replacements. + if not Match(r'\s*;', last_line[end_pos:]): + return + + if linenum == end_line: + expression = lines[linenum][start_pos + 1:end_pos - 1] else: - # Unparenthesized operand. Instead of appending to lhs one character - # at a time, we do another regular expression match to consume several - # characters at once if possible. Trivial benchmark shows that this - # is more efficient when the operands are longer than a single - # character, which is generally the case. - matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) - if not matched: - matched = Match(r'^(\s*\S)(.*)$', expression) - if not matched: - break - lhs += matched.group(1) - expression = matched.group(2) - - # Only apply checks if we got all parts of the boolean expression - if not (lhs and operator and rhs): - return - - # Check that rhs do not contain logical operators. We already know - # that lhs is fine since the loop above parses out && and ||. - if rhs.find('&&') > -1 or rhs.find('||') > -1: - return - - # At least one of the operands must be a constant literal. This is - # to avoid suggesting replacements for unprintable things like - # CHECK(variable != iterator) - # - # The following pattern matches decimal, hex integers, strings, and - # characters (in that order). - lhs = lhs.strip() - rhs = rhs.strip() - match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' - if Match(match_constant, lhs) or Match(match_constant, rhs): - # Note: since we know both lhs and rhs, we can provide a more - # descriptive error message like: - # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) - # Instead of: - # Consider using CHECK_EQ instead of CHECK(a == b) + expression = lines[linenum][start_pos + 1:] + for i in xrange(linenum + 1, end_line): + expression += lines[i] + expression += last_line[0:end_pos - 1] + + # Parse expression so that we can take parentheses into account. + # This avoids false positives for inputs like "CHECK((a < 4) == b)", + # which is not replaceable by CHECK_LE. + lhs = '' + rhs = '' + operator = None + while expression: + matched = Match(r'^\s*(<<|<<=|>>|>>=|->\*|->|&&|\|\||' + r'==|!=|>=|>|<=|<|\()(.*)$', expression) + if matched: + token = matched.group(1) + if token == '(': + # Parenthesized operand + expression = matched.group(2) + (end, _) = FindEndOfExpressionInLine(expression, 0, ['(']) + if end < 0: + return # Unmatched parenthesis + lhs += '(' + expression[0:end] + expression = expression[end:] + elif token in ('&&', '||'): + # Logical and/or operators. This means the expression + # contains more than one term, for example: + # CHECK(42 < a && a < b); + # + # These are not replaceable with CHECK_LE, so bail out early. + return + elif token in ('<<', '<<=', '>>', '>>=', '->*', '->'): + # Non-relational operator + lhs += token + expression = matched.group(2) + else: + # Relational operator + operator = token + rhs = matched.group(2) + break + else: + # Unparenthesized operand. Instead of appending to lhs one character + # at a time, we do another regular expression match to consume several + # characters at once if possible. Trivial benchmark shows that this + # is more efficient when the operands are longer than a single + # character, which is generally the case. + matched = Match(r'^([^-=!<>()&|]+)(.*)$', expression) + if not matched: + matched = Match(r'^(\s*\S)(.*)$', expression) + if not matched: + break + lhs += matched.group(1) + expression = matched.group(2) + + # Only apply checks if we got all parts of the boolean expression + if not (lhs and operator and rhs): + return + + # Check that rhs do not contain logical operators. We already know + # that lhs is fine since the loop above parses out && and ||. + if rhs.find('&&') > -1 or rhs.find('||') > -1: + return + + # At least one of the operands must be a constant literal. This is + # to avoid suggesting replacements for unprintable things like + # CHECK(variable != iterator) # - # We are still keeping the less descriptive message because if lhs - # or rhs gets long, the error message might become unreadable. - error(filename, linenum, 'readability/check', 2, - 'Consider using %s instead of %s(a %s b)' % ( - _CHECK_REPLACEMENT[check_macro][operator], - check_macro, operator)) + # The following pattern matches decimal, hex integers, strings, and + # characters (in that order). + lhs = lhs.strip() + rhs = rhs.strip() + match_constant = r'^([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')$' + if Match(match_constant, lhs) or Match(match_constant, rhs): + # Note: since we know both lhs and rhs, we can provide a more + # descriptive error message like: + # Consider using CHECK_EQ(x, 42) instead of CHECK(x == 42) + # Instead of: + # Consider using CHECK_EQ instead of CHECK(a == b) + # + # We are still keeping the less descriptive message because if lhs + # or rhs gets long, the error message might become unreadable. + error(filename, linenum, 'readability/check', 2, + 'Consider using %s instead of %s(a %s b)' % + (_CHECK_REPLACEMENT[check_macro][operator], check_macro, + operator)) def CheckAltTokens(filename, clean_lines, linenum, error): - """Check alternative keywords being used in boolean expressions. + """Check alternative keywords being used in boolean expressions. Args: filename: The name of the current file. @@ -4334,31 +4365,31 @@ def CheckAltTokens(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] + line = clean_lines.elided[linenum] - # Avoid preprocessor lines - if Match(r'^\s*#', line): - return + # Avoid preprocessor lines + if Match(r'^\s*#', line): + return - # Last ditch effort to avoid multi-line comments. This will not help - # if the comment started before the current line or ended after the - # current line, but it catches most of the false positives. At least, - # it provides a way to workaround this warning for people who use - # multi-line comments in preprocessor macros. - # - # TODO(unknown): remove this once cpplint has better support for - # multi-line comments. - if line.find('/*') >= 0 or line.find('*/') >= 0: - return + # Last ditch effort to avoid multi-line comments. This will not help + # if the comment started before the current line or ended after the + # current line, but it catches most of the false positives. At least, + # it provides a way to workaround this warning for people who use + # multi-line comments in preprocessor macros. + # + # TODO(unknown): remove this once cpplint has better support for + # multi-line comments. + if line.find('/*') >= 0 or line.find('*/') >= 0: + return - for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): - error(filename, linenum, 'readability/alt_tokens', 2, - 'Use operator %s instead of %s' % ( - _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) + for match in _ALT_TOKEN_REPLACEMENT_PATTERN.finditer(line): + error(filename, linenum, 'readability/alt_tokens', 2, + 'Use operator %s instead of %s' % ( + _ALT_TOKEN_REPLACEMENT[match.group(1)], match.group(1))) def GetLineWidth(line): - """Determines the width of the line in column positions. + """Determines the width of the line in column positions. Args: line: A string, which may be a Unicode string. @@ -4367,21 +4398,21 @@ def GetLineWidth(line): The width of the line in column positions, accounting for Unicode combining characters and wide characters. """ - if isinstance(line, unicode): - width = 0 - for uc in unicodedata.normalize('NFC', line): - if unicodedata.east_asian_width(uc) in ('W', 'F'): - width += 2 - elif not unicodedata.combining(uc): - width += 1 - return width - else: - return len(line) + if isinstance(line, unicode): + width = 0 + for uc in unicodedata.normalize('NFC', line): + if unicodedata.east_asian_width(uc) in ('W', 'F'): + width += 2 + elif not unicodedata.combining(uc): + width += 1 + return width + else: + return len(line) def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, error): - """Checks rules from the 'C++ style rules' section of cppguide.html. + """Checks rules from the 'C++ style rules' section of cppguide.html. Most of these rules are hard to test (naming, comment style), but we do what we can. In particular we check for 2-space indents, line lengths, @@ -4397,105 +4428,105 @@ def CheckStyle(filename, clean_lines, linenum, file_extension, nesting_state, error: The function to call with any errors found. """ - # Don't use "elided" lines here, otherwise we can't check commented lines. - # Don't want to use "raw" either, because we don't want to check inside C++11 - # raw strings, - raw_lines = clean_lines.lines_without_raw_strings - line = raw_lines[linenum] - - if line.find('\t') != -1: - error(filename, linenum, 'whitespace/tab', 1, - 'Tab found; better to use spaces') - - # One or three blank spaces at the beginning of the line is weird; it's - # hard to reconcile that with 2-space indents. - # NOTE: here are the conditions rob pike used for his tests. Mine aren't - # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces - # if(RLENGTH > 20) complain = 0; - # if(match($0, " +(error|private|public|protected):")) complain = 0; - # if(match(prev, "&& *$")) complain = 0; - # if(match(prev, "\\|\\| *$")) complain = 0; - # if(match(prev, "[\",=><] *$")) complain = 0; - # if(match($0, " <<")) complain = 0; - # if(match(prev, " +for \\(")) complain = 0; - # if(prevodd && match(prevprev, " +for \\(")) complain = 0; - scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' - classinfo = nesting_state.InnermostClass() - initial_spaces = 0 - cleansed_line = clean_lines.elided[linenum] - while initial_spaces < len(line) and line[initial_spaces] == ' ': - initial_spaces += 1 - if line and line[-1].isspace(): - error(filename, linenum, 'whitespace/end_of_line', 4, - 'Line ends in whitespace. Consider deleting these extra spaces.') - # There are certain situations we allow one space, notably for - # section labels, and also lines containing multi-line raw strings. - elif ((initial_spaces == 1 or initial_spaces == 3) and - not Match(scope_or_label_pattern, cleansed_line) and - not (clean_lines.raw_lines[linenum] != line and - Match(r'^\s*""', line))): - error(filename, linenum, 'whitespace/indent', 3, - 'Weird number of spaces at line-start. ' - 'Are you using a 2-space indent?') - - # Check if the line is a header guard. - is_header_guard = False - if file_extension == 'h': - cppvar = GetHeaderGuardCPPVariable(filename) - if (line.startswith('#ifndef %s' % cppvar) or - line.startswith('#define %s' % cppvar) or - line.startswith('#endif // %s' % cppvar)): - is_header_guard = True - # #include lines and header guards can be long, since there's no clean way to - # split them. - # - # URLs can be long too. It's possible to split these, but it makes them - # harder to cut&paste. - # - # The "$Id:...$" comment may also get very long without it being the - # developers fault. - if (not line.startswith('#include') and not is_header_guard and - not Match(r'^\s*//.*http(s?)://\S*$', line) and - not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): - line_width = GetLineWidth(line) - extended_length = int((_line_length * 1.25)) - if line_width > extended_length: - error(filename, linenum, 'whitespace/line_length', 4, - 'Lines should very rarely be longer than %i characters' % - extended_length) - elif line_width > _line_length: - error(filename, linenum, 'whitespace/line_length', 2, - 'Lines should be <= %i characters long' % _line_length) - - if (cleansed_line.count(';') > 1 and - # for loops are allowed two ;'s (and may run over two lines). - cleansed_line.find('for') == -1 and - (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or - GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and - # It's ok to have many commands in a switch case that fits in 1 line - not ((cleansed_line.find('case ') != -1 or - cleansed_line.find('default:') != -1) and - cleansed_line.find('break;') != -1)): - error(filename, linenum, 'whitespace/newline', 0, - 'More than one command on the same line') - - # Some more style checks - CheckBraces(filename, clean_lines, linenum, error) - CheckTrailingSemicolon(filename, clean_lines, linenum, error) - CheckEmptyBlockBody(filename, clean_lines, linenum, error) - CheckAccess(filename, clean_lines, linenum, nesting_state, error) - CheckSpacing(filename, clean_lines, linenum, nesting_state, error) - CheckOperatorSpacing(filename, clean_lines, linenum, error) - CheckParenthesisSpacing(filename, clean_lines, linenum, error) - CheckCommaSpacing(filename, clean_lines, linenum, error) - CheckBracesSpacing(filename, clean_lines, linenum, error) - CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) - CheckRValueReference(filename, clean_lines, linenum, nesting_state, error) - CheckCheck(filename, clean_lines, linenum, error) - CheckAltTokens(filename, clean_lines, linenum, error) - classinfo = nesting_state.InnermostClass() - if classinfo: - CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) + # Don't use "elided" lines here, otherwise we can't check commented lines. + # Don't want to use "raw" either, because we don't want to check inside C++11 + # raw strings, + raw_lines = clean_lines.lines_without_raw_strings + line = raw_lines[linenum] + + if line.find('\t') != -1: + error(filename, linenum, 'whitespace/tab', 1, + 'Tab found; better to use spaces') + + # One or three blank spaces at the beginning of the line is weird; it's + # hard to reconcile that with 2-space indents. + # NOTE: here are the conditions rob pike used for his tests. Mine aren't + # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces + # if(RLENGTH > 20) complain = 0; + # if(match($0, " +(error|private|public|protected):")) complain = 0; + # if(match(prev, "&& *$")) complain = 0; + # if(match(prev, "\\|\\| *$")) complain = 0; + # if(match(prev, "[\",=><] *$")) complain = 0; + # if(match($0, " <<")) complain = 0; + # if(match(prev, " +for \\(")) complain = 0; + # if(prevodd && match(prevprev, " +for \\(")) complain = 0; + scope_or_label_pattern = r'\s*\w+\s*:\s*\\?$' + classinfo = nesting_state.InnermostClass() + initial_spaces = 0 + cleansed_line = clean_lines.elided[linenum] + while initial_spaces < len(line) and line[initial_spaces] == ' ': + initial_spaces += 1 + if line and line[-1].isspace(): + error(filename, linenum, 'whitespace/end_of_line', 4, + 'Line ends in whitespace. Consider deleting these extra spaces.') + # There are certain situations we allow one space, notably for + # section labels, and also lines containing multi-line raw strings. + elif ((initial_spaces == 1 or initial_spaces == 3) and + not Match(scope_or_label_pattern, cleansed_line) and + not (clean_lines.raw_lines[linenum] != line and + Match(r'^\s*""', line))): + error(filename, linenum, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. ' + 'Are you using a 2-space indent?') + + # Check if the line is a header guard. + is_header_guard = False + if file_extension == 'h': + cppvar = GetHeaderGuardCPPVariable(filename) + if (line.startswith('#ifndef %s' % cppvar) or + line.startswith('#define %s' % cppvar) or + line.startswith('#endif // %s' % cppvar)): + is_header_guard = True + # #include lines and header guards can be long, since there's no clean way to + # split them. + # + # URLs can be long too. It's possible to split these, but it makes them + # harder to cut&paste. + # + # The "$Id:...$" comment may also get very long without it being the + # developers fault. + if (not line.startswith('#include') and not is_header_guard and + not Match(r'^\s*//.*http(s?)://\S*$', line) and + not Match(r'^// \$Id:.*#[0-9]+ \$$', line)): + line_width = GetLineWidth(line) + extended_length = int((_line_length * 1.25)) + if line_width > extended_length: + error(filename, linenum, 'whitespace/line_length', 4, + 'Lines should very rarely be longer than %i characters' % + extended_length) + elif line_width > _line_length: + error(filename, linenum, 'whitespace/line_length', 2, + 'Lines should be <= %i characters long' % _line_length) + + if (cleansed_line.count(';') > 1 and + # for loops are allowed two ;'s (and may run over two lines). + cleansed_line.find('for') == -1 and + (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or + GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and + # It's ok to have many commands in a switch case that fits in 1 line + not ((cleansed_line.find('case ') != -1 or + cleansed_line.find('default:') != -1) and + cleansed_line.find('break;') != -1)): + error(filename, linenum, 'whitespace/newline', 0, + 'More than one command on the same line') + + # Some more style checks + CheckBraces(filename, clean_lines, linenum, error) + CheckTrailingSemicolon(filename, clean_lines, linenum, error) + CheckEmptyBlockBody(filename, clean_lines, linenum, error) + CheckAccess(filename, clean_lines, linenum, nesting_state, error) + CheckSpacing(filename, clean_lines, linenum, nesting_state, error) + CheckOperatorSpacing(filename, clean_lines, linenum, error) + CheckParenthesisSpacing(filename, clean_lines, linenum, error) + CheckCommaSpacing(filename, clean_lines, linenum, error) + CheckBracesSpacing(filename, clean_lines, linenum, error) + CheckSpacingForFunctionCall(filename, clean_lines, linenum, error) + CheckRValueReference(filename, clean_lines, linenum, nesting_state, error) + CheckCheck(filename, clean_lines, linenum, error) + CheckAltTokens(filename, clean_lines, linenum, error) + classinfo = nesting_state.InnermostClass() + if classinfo: + CheckSectionSpacing(filename, clean_lines, classinfo, linenum, error) _RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') @@ -4508,7 +4539,7 @@ _RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') def _DropCommonSuffixes(filename): - """Drops common suffixes like _test.cc or -inl.h from filename. + """Drops common suffixes like _test.cc or -inl.h from filename. For example: >>> _DropCommonSuffixes('foo/foo-inl.h') @@ -4526,16 +4557,16 @@ def _DropCommonSuffixes(filename): Returns: The filename with the common suffix removed. """ - for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', - 'inl.h', 'impl.h', 'internal.h'): - if (filename.endswith(suffix) and len(filename) > len(suffix) and - filename[-len(suffix) - 1] in ('-', '_')): - return filename[:-len(suffix) - 1] - return os.path.splitext(filename)[0] + for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', 'inl.h', 'impl.h', + 'internal.h'): + if (filename.endswith(suffix) and len(filename) > len(suffix) and + filename[-len(suffix) - 1] in ('-', '_')): + return filename[:-len(suffix) - 1] + return os.path.splitext(filename)[0] def _IsTestFilename(filename): - """Determines if the given filename has a suffix that identifies it as a test. + """Determines if the given filename has a suffix that identifies it as a test. Args: filename: The input filename. @@ -4543,16 +4574,15 @@ def _IsTestFilename(filename): Returns: True if 'filename' looks like a test, False otherwise. """ - if (filename.endswith('_test.cc') or - filename.endswith('_unittest.cc') or - filename.endswith('_regtest.cc')): - return True - else: - return False + if (filename.endswith('_test.cc') or filename.endswith('_unittest.cc') or + filename.endswith('_regtest.cc')): + return True + else: + return False def _ClassifyInclude(fileinfo, include, is_system): - """Figures out what kind of header 'include' is. + """Figures out what kind of header 'include' is. Args: fileinfo: The current file cpplint is running over. A FileInfo instance. @@ -4575,44 +4605,43 @@ def _ClassifyInclude(fileinfo, include, is_system): >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) _OTHER_HEADER """ - # This is a list of all standard c++ header files, except - # those already checked for above. - is_cpp_h = include in _CPP_HEADERS - - if is_system: - if is_cpp_h: - return _CPP_SYS_HEADER - else: - return _C_SYS_HEADER - - # If the target file and the include we're checking share a - # basename when we drop common extensions, and the include - # lives in . , then it's likely to be owned by the target file. - target_dir, target_base = ( - os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) - include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) - if target_base == include_base and ( - include_dir == target_dir or - include_dir == os.path.normpath(target_dir + '/../public')): - return _LIKELY_MY_HEADER - - # If the target and include share some initial basename - # component, it's possible the target is implementing the - # include, so it's allowed to be first, but we'll never - # complain if it's not there. - target_first_component = _RE_FIRST_COMPONENT.match(target_base) - include_first_component = _RE_FIRST_COMPONENT.match(include_base) - if (target_first_component and include_first_component and - target_first_component.group(0) == - include_first_component.group(0)): - return _POSSIBLE_MY_HEADER - - return _OTHER_HEADER + # This is a list of all standard c++ header files, except + # those already checked for above. + is_cpp_h = include in _CPP_HEADERS + if is_system: + if is_cpp_h: + return _CPP_SYS_HEADER + else: + return _C_SYS_HEADER + + # If the target file and the include we're checking share a + # basename when we drop common extensions, and the include + # lives in . , then it's likely to be owned by the target file. + target_dir, target_base = ( + os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) + include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) + if target_base == include_base and ( + include_dir == target_dir or + include_dir == os.path.normpath(target_dir + '/../public')): + return _LIKELY_MY_HEADER + + # If the target and include share some initial basename + # component, it's possible the target is implementing the + # include, so it's allowed to be first, but we'll never + # complain if it's not there. + target_first_component = _RE_FIRST_COMPONENT.match(target_base) + include_first_component = _RE_FIRST_COMPONENT.match(include_base) + if (target_first_component and include_first_component and + target_first_component.group(0) == + include_first_component.group(0)): + return _POSSIBLE_MY_HEADER + + return _OTHER_HEADER def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): - """Check rules that are applicable to #include lines. + """Check rules that are applicable to #include lines. Strings on #include lines are NOT removed from elided line, to make certain tasks easier. However, to prevent false positives, checks @@ -4625,68 +4654,69 @@ def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): include_state: An _IncludeState instance in which the headers are inserted. error: The function to call with any errors found. """ - fileinfo = FileInfo(filename) - line = clean_lines.lines[linenum] - - # "include" should use the new style "foo/bar.h" instead of just "bar.h" - # Only do this check if the included header follows google naming - # conventions. If not, assume that it's a 3rd party API that - # requires special include conventions. - # - # We also make an exception for Lua headers, which follow google - # naming convention but not the include convention. - match = Match(r'#include\s*"([^/]+\.h)"', line) - if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): - error(filename, linenum, 'build/include', 4, - 'Include the directory when naming .h files') - - # we shouldn't include a file more than once. actually, there are a - # handful of instances where doing so is okay, but in general it's - # not. - match = _RE_PATTERN_INCLUDE.search(line) - if match: - include = match.group(2) - is_system = (match.group(1) == '<') - duplicate_line = include_state.FindHeader(include) - if duplicate_line >= 0: - error(filename, linenum, 'build/include', 4, - '"%s" already included at %s:%s' % - (include, filename, duplicate_line)) - elif (include.endswith('.cc') and - os.path.dirname(fileinfo.RepositoryName()) != os.path.dirname(include)): - error(filename, linenum, 'build/include', 4, - 'Do not include .cc files from other packages') - elif not _THIRD_PARTY_HEADERS_PATTERN.match(include): - include_state.include_list[-1].append((include, linenum)) - - # We want to ensure that headers appear in the right order: - # 1) for foo.cc, foo.h (preferred location) - # 2) c system files - # 3) cpp system files - # 4) for foo.cc, foo.h (deprecated location) - # 5) other google headers - # - # We classify each include statement as one of those 5 types - # using a number of techniques. The include_state object keeps - # track of the highest type seen, and complains if we see a - # lower type after that. - error_message = include_state.CheckNextIncludeOrder( - _ClassifyInclude(fileinfo, include, is_system)) - if error_message: - error(filename, linenum, 'build/include_order', 4, - '%s. Should be: %s.h, c system, c++ system, other.' % - (error_message, fileinfo.BaseName())) - canonical_include = include_state.CanonicalizeAlphabeticalOrder(include) - if not include_state.IsInAlphabeticalOrder( - clean_lines, linenum, canonical_include): - error(filename, linenum, 'build/include_alpha', 4, - 'Include "%s" not in alphabetical order' % include) - include_state.SetLastHeader(canonical_include) + fileinfo = FileInfo(filename) + line = clean_lines.lines[linenum] + # "include" should use the new style "foo/bar.h" instead of just "bar.h" + # Only do this check if the included header follows google naming + # conventions. If not, assume that it's a 3rd party API that + # requires special include conventions. + # + # We also make an exception for Lua headers, which follow google + # naming convention but not the include convention. + match = Match(r'#include\s*"([^/]+\.h)"', line) + if match and not _THIRD_PARTY_HEADERS_PATTERN.match(match.group(1)): + error(filename, linenum, 'build/include', 4, + 'Include the directory when naming .h files') + + # we shouldn't include a file more than once. actually, there are a + # handful of instances where doing so is okay, but in general it's + # not. + match = _RE_PATTERN_INCLUDE.search(line) + if match: + include = match.group(2) + is_system = (match.group(1) == '<') + duplicate_line = include_state.FindHeader(include) + if duplicate_line >= 0: + error(filename, linenum, 'build/include', 4, + '"%s" already included at %s:%s' % + (include, filename, duplicate_line)) + elif (include.endswith('.cc') and + os.path.dirname(fileinfo.RepositoryName()) != + os.path.dirname(include)): + error(filename, linenum, 'build/include', 4, + 'Do not include .cc files from other packages') + elif not _THIRD_PARTY_HEADERS_PATTERN.match(include): + include_state.include_list[-1].append((include, linenum)) + + # We want to ensure that headers appear in the right order: + # 1) for foo.cc, foo.h (preferred location) + # 2) c system files + # 3) cpp system files + # 4) for foo.cc, foo.h (deprecated location) + # 5) other google headers + # + # We classify each include statement as one of those 5 types + # using a number of techniques. The include_state object keeps + # track of the highest type seen, and complains if we see a + # lower type after that. + error_message = include_state.CheckNextIncludeOrder( + _ClassifyInclude(fileinfo, include, is_system)) + if error_message: + error(filename, linenum, 'build/include_order', 4, + '%s. Should be: %s.h, c system, c++ system, other.' % + (error_message, fileinfo.BaseName())) + canonical_include = include_state.CanonicalizeAlphabeticalOrder( + include) + if not include_state.IsInAlphabeticalOrder(clean_lines, linenum, + canonical_include): + error(filename, linenum, 'build/include_alpha', 4, + 'Include "%s" not in alphabetical order' % include) + include_state.SetLastHeader(canonical_include) def _GetTextInside(text, start_pattern): - r"""Retrieves all the text between matching open and close parentheses. + r"""Retrieves all the text between matching open and close parentheses. Given a string of lines and a regular expression string, retrieve all the text following the expression and between opening punctuation symbols like @@ -4705,40 +4735,40 @@ def _GetTextInside(text, start_pattern): The extracted text. None if either the opening string or ending punctuation could not be found. """ - # TODO(unknown): Audit cpplint.py to see what places could be profitably - # rewritten to use _GetTextInside (and use inferior regexp matching today). - - # Give opening punctuations to get the matching close-punctuations. - matching_punctuation = {'(': ')', '{': '}', '[': ']'} - closing_punctuation = set(matching_punctuation.itervalues()) - - # Find the position to start extracting text. - match = re.search(start_pattern, text, re.M) - if not match: # start_pattern not found in text. - return None - start_position = match.end(0) - - assert start_position > 0, ( - 'start_pattern must ends with an opening punctuation.') - assert text[start_position - 1] in matching_punctuation, ( - 'start_pattern must ends with an opening punctuation.') - # Stack of closing punctuations we expect to have in text after position. - punctuation_stack = [matching_punctuation[text[start_position - 1]]] - position = start_position - while punctuation_stack and position < len(text): - if text[position] == punctuation_stack[-1]: - punctuation_stack.pop() - elif text[position] in closing_punctuation: - # A closing punctuation without matching opening punctuations. - return None - elif text[position] in matching_punctuation: - punctuation_stack.append(matching_punctuation[text[position]]) - position += 1 - if punctuation_stack: - # Opening punctuations left without matching close-punctuations. - return None - # punctuations match. - return text[start_position:position - 1] + # TODO(unknown): Audit cpplint.py to see what places could be profitably + # rewritten to use _GetTextInside (and use inferior regexp matching today). + + # Give opening punctuations to get the matching close-punctuations. + matching_punctuation = {'(': ')', '{': '}', '[': ']'} + closing_punctuation = set(matching_punctuation.itervalues()) + + # Find the position to start extracting text. + match = re.search(start_pattern, text, re.M) + if not match: # start_pattern not found in text. + return None + start_position = match.end(0) + + assert start_position > 0, ( + 'start_pattern must ends with an opening punctuation.') + assert text[start_position - 1] in matching_punctuation, ( + 'start_pattern must ends with an opening punctuation.') + # Stack of closing punctuations we expect to have in text after position. + punctuation_stack = [matching_punctuation[text[start_position - 1]]] + position = start_position + while punctuation_stack and position < len(text): + if text[position] == punctuation_stack[-1]: + punctuation_stack.pop() + elif text[position] in closing_punctuation: + # A closing punctuation without matching opening punctuations. + return None + elif text[position] in matching_punctuation: + punctuation_stack.append(matching_punctuation[text[position]]) + position += 1 + if punctuation_stack: + # Opening punctuations left without matching close-punctuations. + return None + # punctuations match. + return text[start_position:position - 1] # Patterns for matching call-by-reference parameters. @@ -4763,13 +4793,13 @@ _RE_PATTERN_REF_PARAM = re.compile( # A call-by-const-reference parameter either ends with 'const& identifier' # or looks like 'const type& identifier' when 'type' is atomic. _RE_PATTERN_CONST_REF_PARAM = ( - r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + - r'|const\s+' + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') + r'(?:.*\s*\bconst\s*&\s*' + _RE_PATTERN_IDENT + r'|const\s+' + + _RE_PATTERN_TYPE + r'\s*&\s*' + _RE_PATTERN_IDENT + r')') -def CheckLanguage(filename, clean_lines, linenum, file_extension, - include_state, nesting_state, error): - """Checks rules from the 'C++ language rules' section of cppguide.html. +def CheckLanguage(filename, clean_lines, linenum, file_extension, include_state, + nesting_state, error): + """Checks rules from the 'C++ language rules' section of cppguide.html. Some of these rules are hard to test (function overloading, using uint32 inappropriately), but we do the best we can. @@ -4784,149 +4814,152 @@ def CheckLanguage(filename, clean_lines, linenum, file_extension, the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - # If the line is empty or consists of entirely a comment, no need to - # check it. - line = clean_lines.elided[linenum] - if not line: - return - - match = _RE_PATTERN_INCLUDE.search(line) - if match: - CheckIncludeLine(filename, clean_lines, linenum, include_state, error) - return - - # Reset include state across preprocessor directives. This is meant - # to silence warnings for conditional includes. - match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) - if match: - include_state.ResetSection(match.group(1)) - - # Make Windows paths like Unix. - fullname = os.path.abspath(filename).replace('\\', '/') - - # Perform other checks now that we are sure that this is not an include line - CheckCasts(filename, clean_lines, linenum, error) - CheckGlobalStatic(filename, clean_lines, linenum, error) - CheckPrintf(filename, clean_lines, linenum, error) - - if file_extension == 'h': - # TODO(unknown): check that 1-arg constructors are explicit. - # How to tell it's a constructor? - # (handled in CheckForNonStandardConstructs for now) - # TODO(unknown): check that classes declare or disable copy/assign - # (level 1 error) - pass + # If the line is empty or consists of entirely a comment, no need to + # check it. + line = clean_lines.elided[linenum] + if not line: + return - # Check if people are using the verboten C basic types. The only exception - # we regularly allow is "unsigned short port" for port. - if Search(r'\bshort port\b', line): - if not Search(r'\bunsigned short port\b', line): - error(filename, linenum, 'runtime/int', 4, - 'Use "unsigned short" for ports, not "short"') - else: - match = Search(r'\b(short|long(?! +double)|long long)\b', line) + match = _RE_PATTERN_INCLUDE.search(line) if match: - error(filename, linenum, 'runtime/int', 4, - 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) - - # Check if some verboten operator overloading is going on - # TODO(unknown): catch out-of-line unary operator&: - # class X {}; - # int operator&(const X& x) { return 42; } // unary operator& - # The trick is it's hard to tell apart from binary operator&: - # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& - if Search(r'\boperator\s*&\s*\(\s*\)', line): - error(filename, linenum, 'runtime/operator', 4, - 'Unary operator& is dangerous. Do not use it.') - - # Check for suspicious usage of "if" like - # } if (a == b) { - if Search(r'\}\s*if\s*\(', line): - error(filename, linenum, 'readability/braces', 4, - 'Did you mean "else if"? If not, start a new line for "if".') - - # Check for potential format string bugs like printf(foo). - # We constrain the pattern not to pick things like DocidForPrintf(foo). - # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) - # TODO(unknown): Catch the following case. Need to change the calling - # convention of the whole function to process multiple line to handle it. - # printf( - # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); - printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') - if printf_args: - match = Match(r'([\w.\->()]+)$', printf_args) - if match and match.group(1) != '__VA_ARGS__': - function_name = re.search(r'\b((?:string)?printf)\s*\(', - line, re.I).group(1) - error(filename, linenum, 'runtime/printf', 4, - 'Potential format string bug. Do %s("%%s", %s) instead.' - % (function_name, match.group(1))) - - # Check for potential memset bugs like memset(buf, sizeof(buf), 0). - match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) - if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): - error(filename, linenum, 'runtime/memset', 4, - 'Did you mean "memset(%s, 0, %s)"?' - % (match.group(1), match.group(2))) - - if Search(r'\busing namespace\b', line): - error(filename, linenum, 'build/namespaces', 5, - 'Do not use namespace using-directives. ' - 'Use using-declarations instead.') - - # Detect variable-length arrays. - match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) - if (match and match.group(2) != 'return' and match.group(2) != 'delete' and - match.group(3).find(']') == -1): - # Split the size using space and arithmetic operators as delimiters. - # If any of the resulting tokens are not compile time constants then - # report the error. - tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) - is_const = True - skip_next = False - for tok in tokens: - if skip_next: + CheckIncludeLine(filename, clean_lines, linenum, include_state, error) + return + + # Reset include state across preprocessor directives. This is meant + # to silence warnings for conditional includes. + match = Match(r'^\s*#\s*(if|ifdef|ifndef|elif|else|endif)\b', line) + if match: + include_state.ResetSection(match.group(1)) + + # Make Windows paths like Unix. + fullname = os.path.abspath(filename).replace('\\', '/') + + # Perform other checks now that we are sure that this is not an include line + CheckCasts(filename, clean_lines, linenum, error) + CheckGlobalStatic(filename, clean_lines, linenum, error) + CheckPrintf(filename, clean_lines, linenum, error) + + if file_extension == 'h': + # TODO(unknown): check that 1-arg constructors are explicit. + # How to tell it's a constructor? + # (handled in CheckForNonStandardConstructs for now) + # TODO(unknown): check that classes declare or disable copy/assign + # (level 1 error) + pass + + # Check if people are using the verboten C basic types. The only exception + # we regularly allow is "unsigned short port" for port. + if Search(r'\bshort port\b', line): + if not Search(r'\bunsigned short port\b', line): + error(filename, linenum, 'runtime/int', 4, + 'Use "unsigned short" for ports, not "short"') + else: + match = Search(r'\b(short|long(?! +double)|long long)\b', line) + if match: + error(filename, linenum, 'runtime/int', 4, + 'Use int16/int64/etc, rather than the C type %s' % + match.group(1)) + + # Check if some verboten operator overloading is going on + # TODO(unknown): catch out-of-line unary operator&: + # class X {}; + # int operator&(const X& x) { return 42; } // unary operator& + # The trick is it's hard to tell apart from binary operator&: + # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& + if Search(r'\boperator\s*&\s*\(\s*\)', line): + error(filename, linenum, 'runtime/operator', 4, + 'Unary operator& is dangerous. Do not use it.') + + # Check for suspicious usage of "if" like + # } if (a == b) { + if Search(r'\}\s*if\s*\(', line): + error(filename, linenum, 'readability/braces', 4, + 'Did you mean "else if"? If not, start a new line for "if".') + + # Check for potential format string bugs like printf(foo). + # We constrain the pattern not to pick things like DocidForPrintf(foo). + # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) + # TODO(unknown): Catch the following case. Need to change the calling + # convention of the whole function to process multiple line to handle it. + # printf( + # boy_this_is_a_really_long_variable_that_cannot_fit_on_the_prev_line); + printf_args = _GetTextInside(line, r'(?i)\b(string)?printf\s*\(') + if printf_args: + match = Match(r'([\w.\->()]+)$', printf_args) + if match and match.group(1) != '__VA_ARGS__': + function_name = re.search(r'\b((?:string)?printf)\s*\(', line, + re.I).group(1) + error(filename, linenum, 'runtime/printf', 4, + 'Potential format string bug. Do %s("%%s", %s) instead.' % + (function_name, match.group(1))) + + # Check for potential memset bugs like memset(buf, sizeof(buf), 0). + match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) + if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): + error(filename, linenum, 'runtime/memset', 4, + 'Did you mean "memset(%s, 0, %s)"?' % + (match.group(1), match.group(2))) + + if Search(r'\busing namespace\b', line): + error(filename, linenum, 'build/namespaces', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + + # Detect variable-length arrays. + match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) + if (match and match.group(2) != 'return' and match.group(2) != 'delete' and + match.group(3).find(']') == -1): + # Split the size using space and arithmetic operators as delimiters. + # If any of the resulting tokens are not compile time constants then + # report the error. + tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) + is_const = True skip_next = False - continue - - if Search(r'sizeof\(.+\)', tok): continue - if Search(r'arraysize\(\w+\)', tok): continue - - tok = tok.lstrip('(') - tok = tok.rstrip(')') - if not tok: continue - if Match(r'\d+', tok): continue - if Match(r'0[xX][0-9a-fA-F]+', tok): continue - if Match(r'k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue - if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue - # A catch all for tricky sizeof cases, including 'sizeof expression', - # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' - # requires skipping the next token because we split on ' ' and '*'. - if tok.startswith('sizeof'): - skip_next = True - continue - is_const = False - break - if not is_const: - error(filename, linenum, 'runtime/arrays', 1, - 'Do not use variable-length arrays. Use an appropriately named ' - "('k' followed by CamelCase) compile-time constant for the size.") - - # Check for use of unnamed namespaces in header files. Registration - # macros are typically OK, so we allow use of "namespace {" on lines - # that end with backslashes. - if (file_extension == 'h' - and Search(r'\bnamespace\s*{', line) - and line[-1] != '\\'): - error(filename, linenum, 'build/namespaces', 4, - 'Do not use unnamed namespaces in header files. See ' - 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' - ' for more information.') + for tok in tokens: + if skip_next: + skip_next = False + continue + + if Search(r'sizeof\(.+\)', tok): continue + if Search(r'arraysize\(\w+\)', tok): continue + + tok = tok.lstrip('(') + tok = tok.rstrip(')') + if not tok: continue + if Match(r'\d+', tok): continue + if Match(r'0[xX][0-9a-fA-F]+', tok): continue + if Match(r'k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue + # A catch all for tricky sizeof cases, including 'sizeof expression', + # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' + # requires skipping the next token because we split on ' ' and '*'. + if tok.startswith('sizeof'): + skip_next = True + continue + is_const = False + break + if not is_const: + error( + filename, linenum, 'runtime/arrays', 1, + 'Do not use variable-length arrays. Use an appropriately named ' + "('k' followed by CamelCase) compile-time constant for the size." + ) + + # Check for use of unnamed namespaces in header files. Registration + # macros are typically OK, so we allow use of "namespace {" on lines + # that end with backslashes. + if (file_extension == 'h' and Search(r'\bnamespace\s*{', line) and + line[-1] != '\\'): + error( + filename, linenum, 'build/namespaces', 4, + 'Do not use unnamed namespaces in header files. See ' + 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' + ' for more information.') def CheckGlobalStatic(filename, clean_lines, linenum, error): - """Check for unsafe global or static objects. + """Check for unsafe global or static objects. Args: filename: The name of the current file. @@ -4934,51 +4967,50 @@ def CheckGlobalStatic(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Match two lines at a time to support multiline declarations - if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): - line += clean_lines.elided[linenum + 1].strip() - - # Check for people declaring static/global STL strings at the top level. - # This is dangerous because the C++ language does not guarantee that - # globals with constructors are initialized before the first access. - match = Match( - r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', - line) - - # Remove false positives: - # - String pointers (as opposed to values). - # string *pointer - # const string *pointer - # string const *pointer - # string *const pointer - # - # - Functions and template specializations. - # string Function(... - # string Class::Method(... - # - # - Operators. These are matched separately because operator names - # cross non-word boundaries, and trying to match both operators - # and functions at the same time would decrease accuracy of - # matching identifiers. - # string Class::operator*() - if (match and - not Search(r'\bstring\b(\s+const)?\s*\*\s*(const\s+)?\w', line) and - not Search(r'\boperator\W', line) and - not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(3))): - error(filename, linenum, 'runtime/string', 4, - 'For a static/global string constant, use a C style string instead: ' - '"%schar %s[]".' % - (match.group(1), match.group(2))) - - if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): - error(filename, linenum, 'runtime/init', 4, - 'You seem to be initializing a member variable with itself.') + line = clean_lines.elided[linenum] + + # Match two lines at a time to support multiline declarations + if linenum + 1 < clean_lines.NumLines() and not Search(r'[;({]', line): + line += clean_lines.elided[linenum + 1].strip() + + # Check for people declaring static/global STL strings at the top level. + # This is dangerous because the C++ language does not guarantee that + # globals with constructors are initialized before the first access. + match = Match(r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', + line) + + # Remove false positives: + # - String pointers (as opposed to values). + # string *pointer + # const string *pointer + # string const *pointer + # string *const pointer + # + # - Functions and template specializations. + # string Function(... + # string Class::Method(... + # + # - Operators. These are matched separately because operator names + # cross non-word boundaries, and trying to match both operators + # and functions at the same time would decrease accuracy of + # matching identifiers. + # string Class::operator*() + if (match and + not Search(r'\bstring\b(\s+const)?\s*\*\s*(const\s+)?\w', line) and + not Search(r'\boperator\W', line) and not Match( + r'\s*(<.*>)?(::[a-zA-Z0-9_]+)*\s*\(([^"]|$)', match.group(3))): + error( + filename, linenum, 'runtime/string', 4, + 'For a static/global string constant, use a C style string instead: ' + '"%schar %s[]".' % (match.group(1), match.group(2))) + + if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): + error(filename, linenum, 'runtime/init', 4, + 'You seem to be initializing a member variable with itself.') def CheckPrintf(filename, clean_lines, linenum, error): - """Check for printf related issues. + """Check for printf related issues. Args: filename: The name of the current file. @@ -4986,28 +5018,28 @@ def CheckPrintf(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # When snprintf is used, the second argument shouldn't be a literal. - match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) - if match and match.group(2) != '0': - # If 2nd arg is zero, snprintf is used to calculate size. - error(filename, linenum, 'runtime/printf', 3, - 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' - 'to snprintf.' % (match.group(1), match.group(2))) - - # Check if some verboten C functions are being used. - if Search(r'\bsprintf\s*\(', line): - error(filename, linenum, 'runtime/printf', 5, - 'Never use sprintf. Use snprintf instead.') - match = Search(r'\b(strcpy|strcat)\s*\(', line) - if match: - error(filename, linenum, 'runtime/printf', 4, - 'Almost always, snprintf is better than %s' % match.group(1)) + line = clean_lines.elided[linenum] + + # When snprintf is used, the second argument shouldn't be a literal. + match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) + if match and match.group(2) != '0': + # If 2nd arg is zero, snprintf is used to calculate size. + error(filename, linenum, 'runtime/printf', 3, + 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' + 'to snprintf.' % (match.group(1), match.group(2))) + + # Check if some verboten C functions are being used. + if Search(r'\bsprintf\s*\(', line): + error(filename, linenum, 'runtime/printf', 5, + 'Never use sprintf. Use snprintf instead.') + match = Search(r'\b(strcpy|strcat)\s*\(', line) + if match: + error(filename, linenum, 'runtime/printf', 4, + 'Almost always, snprintf is better than %s' % match.group(1)) def IsDerivedFunction(clean_lines, linenum): - """Check if current line contains an inherited function. + """Check if current line contains an inherited function. Args: clean_lines: A CleansedLines instance containing the file. @@ -5016,20 +5048,20 @@ def IsDerivedFunction(clean_lines, linenum): True if current line contains a function with "override" virt-specifier. """ - # Scan back a few lines for start of current function - for i in xrange(linenum, max(-1, linenum - 10), -1): - match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) - if match: - # Look for "override" after the matching closing parenthesis - line, _, closing_paren = CloseExpression( - clean_lines, i, len(match.group(1))) - return (closing_paren >= 0 and - Search(r'\boverride\b', line[closing_paren:])) - return False + # Scan back a few lines for start of current function + for i in xrange(linenum, max(-1, linenum - 10), -1): + match = Match(r'^([^()]*\w+)\(', clean_lines.elided[i]) + if match: + # Look for "override" after the matching closing parenthesis + line, _, closing_paren = CloseExpression(clean_lines, i, + len(match.group(1))) + return (closing_paren >= 0 and + Search(r'\boverride\b', line[closing_paren:])) + return False def IsOutOfLineMethodDefinition(clean_lines, linenum): - """Check if current line contains an out-of-line method definition. + """Check if current line contains an out-of-line method definition. Args: clean_lines: A CleansedLines instance containing the file. @@ -5037,15 +5069,16 @@ def IsOutOfLineMethodDefinition(clean_lines, linenum): Returns: True if current line contains an out-of-line method definition. """ - # Scan back a few lines for start of current function - for i in xrange(linenum, max(-1, linenum - 10), -1): - if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): - return Match(r'^[^()]*\w+::\w+\(', clean_lines.elided[i]) is not None - return False + # Scan back a few lines for start of current function + for i in xrange(linenum, max(-1, linenum - 10), -1): + if Match(r'^([^()]*\w+)\(', clean_lines.elided[i]): + return Match(r'^[^()]*\w+::\w+\(', + clean_lines.elided[i]) is not None + return False def IsInitializerList(clean_lines, linenum): - """Check if current line is inside constructor initializer list. + """Check if current line is inside constructor initializer list. Args: clean_lines: A CleansedLines instance containing the file. @@ -5054,41 +5087,41 @@ def IsInitializerList(clean_lines, linenum): True if current line appears to be inside constructor initializer list, False otherwise. """ - for i in xrange(linenum, 1, -1): - line = clean_lines.elided[i] - if i == linenum: - remove_function_body = Match(r'^(.*)\{\s*$', line) - if remove_function_body: - line = remove_function_body.group(1) - - if Search(r'\s:\s*\w+[({]', line): - # A lone colon tend to indicate the start of a constructor - # initializer list. It could also be a ternary operator, which - # also tend to appear in constructor initializer lists as - # opposed to parameter lists. - return True - if Search(r'\}\s*,\s*$', line): - # A closing brace followed by a comma is probably the end of a - # brace-initialized member in constructor initializer list. - return True - if Search(r'[{};]\s*$', line): - # Found one of the following: - # - A closing brace or semicolon, probably the end of the previous - # function. - # - An opening brace, probably the start of current class or namespace. - # - # Current line is probably not inside an initializer list since - # we saw one of those things without seeing the starting colon. - return False - - # Got to the beginning of the file without seeing the start of - # constructor initializer list. - return False - - -def CheckForNonConstReference(filename, clean_lines, linenum, - nesting_state, error): - """Check for non-const references. + for i in xrange(linenum, 1, -1): + line = clean_lines.elided[i] + if i == linenum: + remove_function_body = Match(r'^(.*)\{\s*$', line) + if remove_function_body: + line = remove_function_body.group(1) + + if Search(r'\s:\s*\w+[({]', line): + # A lone colon tend to indicate the start of a constructor + # initializer list. It could also be a ternary operator, which + # also tend to appear in constructor initializer lists as + # opposed to parameter lists. + return True + if Search(r'\}\s*,\s*$', line): + # A closing brace followed by a comma is probably the end of a + # brace-initialized member in constructor initializer list. + return True + if Search(r'[{};]\s*$', line): + # Found one of the following: + # - A closing brace or semicolon, probably the end of the previous + # function. + # - An opening brace, probably the start of current class or namespace. + # + # Current line is probably not inside an initializer list since + # we saw one of those things without seeing the starting colon. + return False + + # Got to the beginning of the file without seeing the start of + # constructor initializer list. + return False + + +def CheckForNonConstReference(filename, clean_lines, linenum, nesting_state, + error): + """Check for non-const references. Separate from CheckLanguage since it scans backwards from current line, instead of scanning forward. @@ -5101,131 +5134,131 @@ def CheckForNonConstReference(filename, clean_lines, linenum, the current stack of nested blocks being parsed. error: The function to call with any errors found. """ - # Do nothing if there is no '&' on current line. - line = clean_lines.elided[linenum] - if '&' not in line: - return - - # If a function is inherited, current function doesn't have much of - # a choice, so any non-const references should not be blamed on - # derived function. - if IsDerivedFunction(clean_lines, linenum): - return - - # Don't warn on out-of-line method definitions, as we would warn on the - # in-line declaration, if it isn't marked with 'override'. - if IsOutOfLineMethodDefinition(clean_lines, linenum): - return - - # Long type names may be broken across multiple lines, usually in one - # of these forms: - # LongType - # ::LongTypeContinued &identifier - # LongType:: - # LongTypeContinued &identifier - # LongType< - # ...>::LongTypeContinued &identifier - # - # If we detected a type split across two lines, join the previous - # line to current line so that we can match const references - # accordingly. - # - # Note that this only scans back one line, since scanning back - # arbitrary number of lines would be expensive. If you have a type - # that spans more than 2 lines, please use a typedef. - if linenum > 1: - previous = None - if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): - # previous_line\n + ::current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', - clean_lines.elided[linenum - 1]) - elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): - # previous_line::\n + current_line - previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', - clean_lines.elided[linenum - 1]) - if previous: - line = previous.group(1) + line.lstrip() - else: - # Check for templated parameter that is split across multiple lines - endpos = line.rfind('>') - if endpos > -1: - (_, startline, startpos) = ReverseCloseExpression( - clean_lines, linenum, endpos) - if startpos > -1 and startline < linenum: - # Found the matching < on an earlier line, collect all - # pieces up to current line. - line = '' - for i in xrange(startline, linenum + 1): - line += clean_lines.elided[i].strip() - - # Check for non-const references in function parameters. A single '&' may - # found in the following places: - # inside expression: binary & for bitwise AND - # inside expression: unary & for taking the address of something - # inside declarators: reference parameter - # We will exclude the first two cases by checking that we are not inside a - # function body, including one that was just introduced by a trailing '{'. - # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. - if (nesting_state.previous_stack_top and - not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or - isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): - # Not at toplevel, not within a class, and not within a namespace - return - - # Avoid initializer lists. We only need to scan back from the - # current line for something that starts with ':'. - # - # We don't need to check the current line, since the '&' would - # appear inside the second set of parentheses on the current line as - # opposed to the first set. - if linenum > 0: - for i in xrange(linenum - 1, max(0, linenum - 10), -1): - previous_line = clean_lines.elided[i] - if not Search(r'[),]\s*$', previous_line): - break - if Match(r'^\s*:\s+\S', previous_line): + # Do nothing if there is no '&' on current line. + line = clean_lines.elided[linenum] + if '&' not in line: + return + + # If a function is inherited, current function doesn't have much of + # a choice, so any non-const references should not be blamed on + # derived function. + if IsDerivedFunction(clean_lines, linenum): + return + + # Don't warn on out-of-line method definitions, as we would warn on the + # in-line declaration, if it isn't marked with 'override'. + if IsOutOfLineMethodDefinition(clean_lines, linenum): + return + + # Long type names may be broken across multiple lines, usually in one + # of these forms: + # LongType + # ::LongTypeContinued &identifier + # LongType:: + # LongTypeContinued &identifier + # LongType< + # ...>::LongTypeContinued &identifier + # + # If we detected a type split across two lines, join the previous + # line to current line so that we can match const references + # accordingly. + # + # Note that this only scans back one line, since scanning back + # arbitrary number of lines would be expensive. If you have a type + # that spans more than 2 lines, please use a typedef. + if linenum > 1: + previous = None + if Match(r'\s*::(?:[\w<>]|::)+\s*&\s*\S', line): + # previous_line\n + ::current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+[\w<>])\s*$', + clean_lines.elided[linenum - 1]) + elif Match(r'\s*[a-zA-Z_]([\w<>]|::)+\s*&\s*\S', line): + # previous_line::\n + current_line + previous = Search(r'\b((?:const\s*)?(?:[\w<>]|::)+::)\s*$', + clean_lines.elided[linenum - 1]) + if previous: + line = previous.group(1) + line.lstrip() + else: + # Check for templated parameter that is split across multiple lines + endpos = line.rfind('>') + if endpos > -1: + (_, startline, startpos) = ReverseCloseExpression( + clean_lines, linenum, endpos) + if startpos > -1 and startline < linenum: + # Found the matching < on an earlier line, collect all + # pieces up to current line. + line = '' + for i in xrange(startline, linenum + 1): + line += clean_lines.elided[i].strip() + + # Check for non-const references in function parameters. A single '&' may + # found in the following places: + # inside expression: binary & for bitwise AND + # inside expression: unary & for taking the address of something + # inside declarators: reference parameter + # We will exclude the first two cases by checking that we are not inside a + # function body, including one that was just introduced by a trailing '{'. + # TODO(unknown): Doesn't account for 'catch(Exception& e)' [rare]. + if (nesting_state.previous_stack_top and + not (isinstance(nesting_state.previous_stack_top, _ClassInfo) or + isinstance(nesting_state.previous_stack_top, _NamespaceInfo))): + # Not at toplevel, not within a class, and not within a namespace + return + + # Avoid initializer lists. We only need to scan back from the + # current line for something that starts with ':'. + # + # We don't need to check the current line, since the '&' would + # appear inside the second set of parentheses on the current line as + # opposed to the first set. + if linenum > 0: + for i in xrange(linenum - 1, max(0, linenum - 10), -1): + previous_line = clean_lines.elided[i] + if not Search(r'[),]\s*$', previous_line): + break + if Match(r'^\s*:\s+\S', previous_line): + return + + # Avoid preprocessors + if Search(r'\\\s*$', line): return - # Avoid preprocessors - if Search(r'\\\s*$', line): - return - - # Avoid constructor initializer lists - if IsInitializerList(clean_lines, linenum): - return - - # We allow non-const references in a few standard places, like functions - # called "swap()" or iostream operators like "<<" or ">>". Do not check - # those function parameters. - # - # We also accept & in static_assert, which looks like a function but - # it's actually a declaration expression. - whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' - r'operator\s*[<>][<>]|' - r'static_assert|COMPILE_ASSERT' - r')\s*\(') - if Search(whitelisted_functions, line): - return - elif not Search(r'\S+\([^)]*$', line): - # Don't see a whitelisted function on this line. Actually we - # didn't see any function name on this line, so this is likely a - # multi-line parameter list. Try a bit harder to catch this case. - for i in xrange(2): - if (linenum > i and - Search(whitelisted_functions, clean_lines.elided[linenum - i - 1])): + # Avoid constructor initializer lists + if IsInitializerList(clean_lines, linenum): return - decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body - for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): - if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter): - error(filename, linenum, 'runtime/references', 2, - 'Is this a non-const reference? ' - 'If so, make const or use a pointer: ' + - ReplaceAll(' *<', '<', parameter)) + # We allow non-const references in a few standard places, like functions + # called "swap()" or iostream operators like "<<" or ">>". Do not check + # those function parameters. + # + # We also accept & in static_assert, which looks like a function but + # it's actually a declaration expression. + whitelisted_functions = (r'(?:[sS]wap(?:<\w:+>)?|' + r'operator\s*[<>][<>]|' + r'static_assert|COMPILE_ASSERT' + r')\s*\(') + if Search(whitelisted_functions, line): + return + elif not Search(r'\S+\([^)]*$', line): + # Don't see a whitelisted function on this line. Actually we + # didn't see any function name on this line, so this is likely a + # multi-line parameter list. Try a bit harder to catch this case. + for i in xrange(2): + if (linenum > i and Search(whitelisted_functions, + clean_lines.elided[linenum - i - 1])): + return + + decls = ReplaceAll(r'{[^}]*}', ' ', line) # exclude function body + for parameter in re.findall(_RE_PATTERN_REF_PARAM, decls): + if not Match(_RE_PATTERN_CONST_REF_PARAM, parameter): + error(filename, linenum, 'runtime/references', 2, + 'Is this a non-const reference? ' + 'If so, make const or use a pointer: ' + ReplaceAll( + ' *<', '<', parameter)) def CheckCasts(filename, clean_lines, linenum, error): - """Various cast related checks. + """Various cast related checks. Args: filename: The name of the current file. @@ -5233,118 +5266,116 @@ def CheckCasts(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Check to see if they're using an conversion function cast. - # I just try to capture the most common basic types, though there are more. - # Parameterless conversion functions, such as bool(), are allowed as they are - # probably a member operator declaration or default constructor. - match = Search( - r'(\bnew\s+|\S<\s*(?:const\s+)?)?\b' - r'(int|float|double|bool|char|int32|uint32|int64|uint64)' - r'(\([^)].*)', line) - expecting_function = ExpectingFunctionArgs(clean_lines, linenum) - if match and not expecting_function: - matched_type = match.group(2) - - # matched_new_or_template is used to silence two false positives: - # - New operators - # - Template arguments with function types + line = clean_lines.elided[linenum] + + # Check to see if they're using an conversion function cast. + # I just try to capture the most common basic types, though there are more. + # Parameterless conversion functions, such as bool(), are allowed as they are + # probably a member operator declaration or default constructor. + match = Search(r'(\bnew\s+|\S<\s*(?:const\s+)?)?\b' + r'(int|float|double|bool|char|int32|uint32|int64|uint64)' + r'(\([^)].*)', line) + expecting_function = ExpectingFunctionArgs(clean_lines, linenum) + if match and not expecting_function: + matched_type = match.group(2) + + # matched_new_or_template is used to silence two false positives: + # - New operators + # - Template arguments with function types + # + # For template arguments, we match on types immediately following + # an opening bracket without any spaces. This is a fast way to + # silence the common case where the function type is the first + # template argument. False negative with less-than comparison is + # avoided because those operators are usually followed by a space. + # + # function // bracket + no space = false positive + # value < double(42) // bracket + space = true positive + matched_new_or_template = match.group(1) + + # Avoid arrays by looking for brackets that come after the closing + # parenthesis. + if Match(r'\([^()]+\)\s*\[', match.group(3)): + return + + # Other things to ignore: + # - Function pointers + # - Casts to pointer types + # - Placement new + # - Alias declarations + matched_funcptr = match.group(3) + if (matched_new_or_template is None and not (matched_funcptr and (Match( + r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', + matched_funcptr) or matched_funcptr.startswith('(*)'))) and + not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and + not Search(r'new\(\S+\)\s*' + matched_type, line)): + error(filename, linenum, 'readability/casting', 4, + 'Using deprecated casting style. ' + 'Use static_cast<%s>(...) instead' % matched_type) + + if not expecting_function: + CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', + r'\((int|float|double|bool|char|u?int(16|32|64))\)', + error) + + # This doesn't catch all cases. Consider (const char * const)"hello". # - # For template arguments, we match on types immediately following - # an opening bracket without any spaces. This is a fast way to - # silence the common case where the function type is the first - # template argument. False negative with less-than comparison is - # avoided because those operators are usually followed by a space. + # (char *) "foo" should always be a const_cast (reinterpret_cast won't + # compile). + if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', + r'\((char\s?\*+\s?)\)\s*"', error): + pass + else: + # Check pointer casts for other than string constants + CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', + r'\((\w+\s?\*+\s?)\)', error) + + # In addition, we look for people taking the address of a cast. This + # is dangerous -- casts can assign to temporaries, so the pointer doesn't + # point where you think. # - # function // bracket + no space = false positive - # value < double(42) // bracket + space = true positive - matched_new_or_template = match.group(1) - - # Avoid arrays by looking for brackets that come after the closing - # parenthesis. - if Match(r'\([^()]+\)\s*\[', match.group(3)): - return - - # Other things to ignore: - # - Function pointers - # - Casts to pointer types - # - Placement new - # - Alias declarations - matched_funcptr = match.group(3) - if (matched_new_or_template is None and - not (matched_funcptr and - (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(', - matched_funcptr) or - matched_funcptr.startswith('(*)'))) and - not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and - not Search(r'new\(\S+\)\s*' + matched_type, line)): - error(filename, linenum, 'readability/casting', 4, - 'Using deprecated casting style. ' - 'Use static_cast<%s>(...) instead' % - matched_type) - - if not expecting_function: - CheckCStyleCast(filename, clean_lines, linenum, 'static_cast', - r'\((int|float|double|bool|char|u?int(16|32|64))\)', error) - - # This doesn't catch all cases. Consider (const char * const)"hello". - # - # (char *) "foo" should always be a const_cast (reinterpret_cast won't - # compile). - if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast', - r'\((char\s?\*+\s?)\)\s*"', error): - pass - else: - # Check pointer casts for other than string constants - CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast', - r'\((\w+\s?\*+\s?)\)', error) - - # In addition, we look for people taking the address of a cast. This - # is dangerous -- casts can assign to temporaries, so the pointer doesn't - # point where you think. - # - # Some non-identifier character is required before the '&' for the - # expression to be recognized as a cast. These are casts: - # expression = &static_cast(temporary()); - # function(&(int*)(temporary())); - # - # This is not a cast: - # reference_type&(int* function_param); - match = Search( - r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' - r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) - if match: - # Try a better error message when the & is bound to something - # dereferenced by the casted pointer, as opposed to the casted - # pointer itself. - parenthesis_error = False - match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line) + # Some non-identifier character is required before the '&' for the + # expression to be recognized as a cast. These are casts: + # expression = &static_cast(temporary()); + # function(&(int*)(temporary())); + # + # This is not a cast: + # reference_type&(int* function_param); + match = Search(r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|' + r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line) if match: - _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1))) - if x1 >= 0 and clean_lines.elided[y1][x1] == '(': - _, y2, x2 = CloseExpression(clean_lines, y1, x1) - if x2 >= 0: - extended_line = clean_lines.elided[y2][x2:] - if y2 < clean_lines.NumLines() - 1: - extended_line += clean_lines.elided[y2 + 1] - if Match(r'\s*(?:->|\[)', extended_line): - parenthesis_error = True - - if parenthesis_error: - error(filename, linenum, 'readability/casting', 4, - ('Are you taking an address of something dereferenced ' - 'from a cast? Wrapping the dereferenced expression in ' - 'parentheses will make the binding more obvious')) - else: - error(filename, linenum, 'runtime/casting', 4, - ('Are you taking an address of a cast? ' - 'This is dangerous: could be a temp var. ' - 'Take the address before doing the cast, rather than after')) + # Try a better error message when the & is bound to something + # dereferenced by the casted pointer, as opposed to the casted + # pointer itself. + parenthesis_error = False + match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', + line) + if match: + _, y1, x1 = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if x1 >= 0 and clean_lines.elided[y1][x1] == '(': + _, y2, x2 = CloseExpression(clean_lines, y1, x1) + if x2 >= 0: + extended_line = clean_lines.elided[y2][x2:] + if y2 < clean_lines.NumLines() - 1: + extended_line += clean_lines.elided[y2 + 1] + if Match(r'\s*(?:->|\[)', extended_line): + parenthesis_error = True + + if parenthesis_error: + error(filename, linenum, 'readability/casting', 4, + ('Are you taking an address of something dereferenced ' + 'from a cast? Wrapping the dereferenced expression in ' + 'parentheses will make the binding more obvious')) + else: + error(filename, linenum, 'runtime/casting', 4, + ('Are you taking an address of a cast? ' + 'This is dangerous: could be a temp var. ' + 'Take the address before doing the cast, rather than after')) def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): - """Checks for a C-style cast by looking for the pattern. + """Checks for a C-style cast by looking for the pattern. Args: filename: The name of the current file. @@ -5359,96 +5390,96 @@ def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error): True if an error was emitted. False otherwise. """ - line = clean_lines.elided[linenum] - match = Search(pattern, line) - if not match: - return False + line = clean_lines.elided[linenum] + match = Search(pattern, line) + if not match: + return False - # Exclude lines with keywords that tend to look like casts - context = line[0:match.start(1) - 1] - if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): - return False + # Exclude lines with keywords that tend to look like casts + context = line[0:match.start(1) - 1] + if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context): + return False - # Try expanding current context to see if we one level of - # parentheses inside a macro. - if linenum > 0: - for i in xrange(linenum - 1, max(0, linenum - 5), -1): - context = clean_lines.elided[i] + context - if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): - return False + # Try expanding current context to see if we one level of + # parentheses inside a macro. + if linenum > 0: + for i in xrange(linenum - 1, max(0, linenum - 5), -1): + context = clean_lines.elided[i] + context + if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context): + return False - # operator++(int) and operator--(int) - if context.endswith(' operator++') or context.endswith(' operator--'): - return False + # operator++(int) and operator--(int) + if context.endswith(' operator++') or context.endswith(' operator--'): + return False - # A single unnamed argument for a function tends to look like old - # style cast. If we see those, don't issue warnings for deprecated - # casts, instead issue warnings for unnamed arguments where - # appropriate. - # - # These are things that we want warnings for, since the style guide - # explicitly require all parameters to be named: - # Function(int); - # Function(int) { - # ConstMember(int) const; - # ConstMember(int) const { - # ExceptionMember(int) throw (...); - # ExceptionMember(int) throw (...) { - # PureVirtual(int) = 0; - # [](int) -> bool { - # - # These are functions of some sort, where the compiler would be fine - # if they had named parameters, but people often omit those - # identifiers to reduce clutter: - # (FunctionPointer)(int); - # (FunctionPointer)(int) = value; - # Function((function_pointer_arg)(int)) - # Function((function_pointer_arg)(int), int param) - # ; - # <(FunctionPointerTemplateArgument)(int)>; - remainder = line[match.end(0):] - if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', - remainder): - # Looks like an unnamed parameter. - - # Don't warn on any kind of template arguments. - if Match(r'^\s*>', remainder): - return False - - # Don't warn on assignments to function pointers, but keep warnings for - # unnamed parameters to pure virtual functions. Note that this pattern - # will also pass on assignments of "0" to function pointers, but the - # preferred values for those would be "nullptr" or "NULL". - matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder) - if matched_zero and matched_zero.group(1) != '0': - return False - - # Don't warn on function pointer declarations. For this we need - # to check what came before the "(type)" string. - if Match(r'.*\)\s*$', line[0:match.start(0)]): - return False - - # Don't warn if the parameter is named with block comments, e.g.: - # Function(int /*unused_param*/); - raw_line = clean_lines.raw_lines[linenum] - if '/*' in raw_line: - return False - - # Passed all filters, issue warning here. - error(filename, linenum, 'readability/function', 3, - 'All parameters should be named in a function') - return True + # A single unnamed argument for a function tends to look like old + # style cast. If we see those, don't issue warnings for deprecated + # casts, instead issue warnings for unnamed arguments where + # appropriate. + # + # These are things that we want warnings for, since the style guide + # explicitly require all parameters to be named: + # Function(int); + # Function(int) { + # ConstMember(int) const; + # ConstMember(int) const { + # ExceptionMember(int) throw (...); + # ExceptionMember(int) throw (...) { + # PureVirtual(int) = 0; + # [](int) -> bool { + # + # These are functions of some sort, where the compiler would be fine + # if they had named parameters, but people often omit those + # identifiers to reduce clutter: + # (FunctionPointer)(int); + # (FunctionPointer)(int) = value; + # Function((function_pointer_arg)(int)) + # Function((function_pointer_arg)(int), int param) + # ; + # <(FunctionPointerTemplateArgument)(int)>; + remainder = line[match.end(0):] + if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)', + remainder): + # Looks like an unnamed parameter. + + # Don't warn on any kind of template arguments. + if Match(r'^\s*>', remainder): + return False - # At this point, all that should be left is actual casts. - error(filename, linenum, 'readability/casting', 4, - 'Using C-style cast. Use %s<%s>(...) instead' % - (cast_type, match.group(1))) + # Don't warn on assignments to function pointers, but keep warnings for + # unnamed parameters to pure virtual functions. Note that this pattern + # will also pass on assignments of "0" to function pointers, but the + # preferred values for those would be "nullptr" or "NULL". + matched_zero = Match(r'^\s=\s*(\S+)\s*;', remainder) + if matched_zero and matched_zero.group(1) != '0': + return False - return True + # Don't warn on function pointer declarations. For this we need + # to check what came before the "(type)" string. + if Match(r'.*\)\s*$', line[0:match.start(0)]): + return False + + # Don't warn if the parameter is named with block comments, e.g.: + # Function(int /*unused_param*/); + raw_line = clean_lines.raw_lines[linenum] + if '/*' in raw_line: + return False + + # Passed all filters, issue warning here. + error(filename, linenum, 'readability/function', 3, + 'All parameters should be named in a function') + return True + + # At this point, all that should be left is actual casts. + error(filename, linenum, 'readability/casting', 4, + 'Using C-style cast. Use %s<%s>(...) instead' % + (cast_type, match.group(1))) + + return True def ExpectingFunctionArgs(clean_lines, linenum): - """Checks whether where function type arguments are expected. + """Checks whether where function type arguments are expected. Args: clean_lines: A CleansedLines instance containing the file. @@ -5458,78 +5489,107 @@ def ExpectingFunctionArgs(clean_lines, linenum): True if the line at 'linenum' is inside something that expects arguments of function types. """ - line = clean_lines.elided[linenum] - return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or - (linenum >= 2 and - (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', - clean_lines.elided[linenum - 1]) or - Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', - clean_lines.elided[linenum - 2]) or - Search(r'\bstd::m?function\s*\<\s*$', - clean_lines.elided[linenum - 1])))) + line = clean_lines.elided[linenum] + return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or + (linenum >= 2 and + (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$', + clean_lines.elided[linenum - 1]) or + Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$', + clean_lines.elided[linenum - 2]) or + Search(r'\bstd::m?function\s*\<\s*$', + clean_lines.elided[linenum - 1])))) _HEADERS_CONTAINING_TEMPLATES = ( - ('', ('deque',)), - ('', ('unary_function', 'binary_function', - 'plus', 'minus', 'multiplies', 'divides', 'modulus', - 'negate', - 'equal_to', 'not_equal_to', 'greater', 'less', - 'greater_equal', 'less_equal', - 'logical_and', 'logical_or', 'logical_not', - 'unary_negate', 'not1', 'binary_negate', 'not2', - 'bind1st', 'bind2nd', - 'pointer_to_unary_function', - 'pointer_to_binary_function', - 'ptr_fun', - 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', - 'mem_fun_ref_t', - 'const_mem_fun_t', 'const_mem_fun1_t', - 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', - 'mem_fun_ref', - )), - ('', ('numeric_limits',)), - ('', ('list',)), - ('', ('map', 'multimap',)), - ('', ('allocator',)), - ('', ('queue', 'priority_queue',)), - ('', ('set', 'multiset',)), - ('', ('stack',)), - ('', ('char_traits', 'basic_string',)), - ('', ('tuple',)), - ('', ('pair',)), - ('', ('vector',)), + ('', ('deque', )), + ('', ( + 'unary_function', + 'binary_function', + 'plus', + 'minus', + 'multiplies', + 'divides', + 'modulus', + 'negate', + 'equal_to', + 'not_equal_to', + 'greater', + 'less', + 'greater_equal', + 'less_equal', + 'logical_and', + 'logical_or', + 'logical_not', + 'unary_negate', + 'not1', + 'binary_negate', + 'not2', + 'bind1st', + 'bind2nd', + 'pointer_to_unary_function', + 'pointer_to_binary_function', + 'ptr_fun', + 'mem_fun_t', + 'mem_fun', + 'mem_fun1_t', + 'mem_fun1_ref_t', + 'mem_fun_ref_t', + 'const_mem_fun_t', + 'const_mem_fun1_t', + 'const_mem_fun_ref_t', + 'const_mem_fun1_ref_t', + 'mem_fun_ref', )), + ('', ('numeric_limits', )), + ('', ('list', )), + ('', ( + 'map', + 'multimap', )), + ('', ('allocator', )), + ('', ( + 'queue', + 'priority_queue', )), + ('', ( + 'set', + 'multiset', )), + ('', ('stack', )), + ('', ( + 'char_traits', + 'basic_string', )), + ('', ('tuple', )), + ('', ('pair', )), + ('', ('vector', )), # gcc extensions. # Note: std::hash is their hash, ::hash is our hash - ('', ('hash_map', 'hash_multimap',)), - ('', ('hash_set', 'hash_multiset',)), - ('', ('slist',)), - ) + ('', ( + 'hash_map', + 'hash_multimap', )), + ('', ( + 'hash_set', + 'hash_multiset', )), + ('', ('slist', )), ) _RE_PATTERN_STRING = re.compile(r'\bstring\b') _re_pattern_algorithm_header = [] for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap', 'transform'): - # Match max(..., ...), max(..., ...), but not foo->max, foo.max or - # type::max(). - _re_pattern_algorithm_header.append( - (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), - _template, - '')) + # Match max(..., ...), max(..., ...), but not foo->max, foo.max or + # type::max(). + _re_pattern_algorithm_header.append( + (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), _template, + '')) _re_pattern_templates = [] for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: - for _template in _templates: - _re_pattern_templates.append( - (re.compile(r'(\<|\b)' + _template + r'\s*\<'), - _template + '<>', - _header)) + for _template in _templates: + _re_pattern_templates.append( + (re.compile(r'(\<|\b)' + _template + r'\s*\<'), _template + '<>', + _header)) def FilesBelongToSameModule(filename_cc, filename_h): - """Check if these two filenames belong to the same module. + """Check if these two filenames belong to the same module. The concept of a 'module' here is a as follows: foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the @@ -5558,33 +5618,33 @@ def FilesBelongToSameModule(filename_cc, filename_h): string: the additional prefix needed to open the header file. """ - if not filename_cc.endswith('.cc'): - return (False, '') - filename_cc = filename_cc[:-len('.cc')] - if filename_cc.endswith('_unittest'): - filename_cc = filename_cc[:-len('_unittest')] - elif filename_cc.endswith('_test'): - filename_cc = filename_cc[:-len('_test')] - filename_cc = filename_cc.replace('/public/', '/') - filename_cc = filename_cc.replace('/internal/', '/') - - if not filename_h.endswith('.h'): - return (False, '') - filename_h = filename_h[:-len('.h')] - if filename_h.endswith('-inl'): - filename_h = filename_h[:-len('-inl')] - filename_h = filename_h.replace('/public/', '/') - filename_h = filename_h.replace('/internal/', '/') - - files_belong_to_same_module = filename_cc.endswith(filename_h) - common_path = '' - if files_belong_to_same_module: - common_path = filename_cc[:-len(filename_h)] - return files_belong_to_same_module, common_path + if not filename_cc.endswith('.cc'): + return (False, '') + filename_cc = filename_cc[:-len('.cc')] + if filename_cc.endswith('_unittest'): + filename_cc = filename_cc[:-len('_unittest')] + elif filename_cc.endswith('_test'): + filename_cc = filename_cc[:-len('_test')] + filename_cc = filename_cc.replace('/public/', '/') + filename_cc = filename_cc.replace('/internal/', '/') + + if not filename_h.endswith('.h'): + return (False, '') + filename_h = filename_h[:-len('.h')] + if filename_h.endswith('-inl'): + filename_h = filename_h[:-len('-inl')] + filename_h = filename_h.replace('/public/', '/') + filename_h = filename_h.replace('/internal/', '/') + + files_belong_to_same_module = filename_cc.endswith(filename_h) + common_path = '' + if files_belong_to_same_module: + common_path = filename_cc[:-len(filename_h)] + return files_belong_to_same_module, common_path def UpdateIncludeState(filename, include_dict, io=codecs): - """Fill up the include_dict with new includes found from the file. + """Fill up the include_dict with new includes found from the file. Args: filename: the name of the header to read. @@ -5594,25 +5654,28 @@ def UpdateIncludeState(filename, include_dict, io=codecs): Returns: True if a header was successfully added. False otherwise. """ - headerfile = None - try: - headerfile = io.open(filename, 'r', 'utf8', 'replace') - except IOError: - return False - linenum = 0 - for line in headerfile: - linenum += 1 - clean_line = CleanseComments(line) - match = _RE_PATTERN_INCLUDE.search(clean_line) - if match: - include = match.group(2) - include_dict.setdefault(include, linenum) - return True + headerfile = None + try: + headerfile = io.open(filename, 'r', 'utf8', 'replace') + except IOError: + return False + linenum = 0 + for line in headerfile: + linenum += 1 + clean_line = CleanseComments(line) + match = _RE_PATTERN_INCLUDE.search(clean_line) + if match: + include = match.group(2) + include_dict.setdefault(include, linenum) + return True -def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, +def CheckForIncludeWhatYouUse(filename, + clean_lines, + include_state, + error, io=codecs): - """Reports for missing stl includes. + """Reports for missing stl includes. This function will output warnings to make sure you are including the headers necessary for the stl containers and functions that you use. We only give one @@ -5628,87 +5691,88 @@ def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, io: The IO factory to use to read the header file. Provided for unittest injection. """ - required = {} # A map of header name to linenumber and the template entity. - # Example of required: { '': (1219, 'less<>') } + required = {} # A map of header name to linenumber and the template entity. + # Example of required: { '': (1219, 'less<>') } - for linenum in xrange(clean_lines.NumLines()): - line = clean_lines.elided[linenum] - if not line or line[0] == '#': - continue + for linenum in xrange(clean_lines.NumLines()): + line = clean_lines.elided[linenum] + if not line or line[0] == '#': + continue - # String is special -- it is a non-templatized type in STL. - matched = _RE_PATTERN_STRING.search(line) - if matched: - # Don't warn about strings in non-STL namespaces: - # (We check only the first match per line; good enough.) - prefix = line[:matched.start()] - if prefix.endswith('std::') or not prefix.endswith('::'): - required[''] = (linenum, 'string') - - for pattern, template, header in _re_pattern_algorithm_header: - if pattern.search(line): - required[header] = (linenum, template) - - # The following function is just a speed up, no semantics are changed. - if not '<' in line: # Reduces the cpu time usage by skipping lines. - continue - - for pattern, template, header in _re_pattern_templates: - if pattern.search(line): - required[header] = (linenum, template) - - # The policy is that if you #include something in foo.h you don't need to - # include it again in foo.cc. Here, we will look at possible includes. - # Let's flatten the include_state include_list and copy it into a dictionary. - include_dict = dict([item for sublist in include_state.include_list - for item in sublist]) - - # Did we find the header for this file (if any) and successfully load it? - header_found = False - - # Use the absolute path so that matching works properly. - abs_filename = FileInfo(filename).FullName() - - # For Emacs's flymake. - # If cpplint is invoked from Emacs's flymake, a temporary file is generated - # by flymake and that file name might end with '_flymake.cc'. In that case, - # restore original file name here so that the corresponding header file can be - # found. - # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' - # instead of 'foo_flymake.h' - abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) - - # include_dict is modified during iteration, so we iterate over a copy of - # the keys. - header_keys = include_dict.keys() - for header in header_keys: - (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) - fullpath = common_path + header - if same_module and UpdateIncludeState(fullpath, include_dict, io): - header_found = True - - # If we can't find the header file for a .cc, assume it's because we don't - # know where to look. In that case we'll give up as we're not sure they - # didn't include it in the .h file. - # TODO(unknown): Do a better job of finding .h files so we are confident that - # not having the .h file means there isn't one. - if filename.endswith('.cc') and not header_found: - return - - # All the lines have been processed, report the errors found. - for required_header_unstripped in required: - template = required[required_header_unstripped][1] - if required_header_unstripped.strip('<>"') not in include_dict: - error(filename, required[required_header_unstripped][0], - 'build/include_what_you_use', 4, - 'Add #include ' + required_header_unstripped + ' for ' + template) + # String is special -- it is a non-templatized type in STL. + matched = _RE_PATTERN_STRING.search(line) + if matched: + # Don't warn about strings in non-STL namespaces: + # (We check only the first match per line; good enough.) + prefix = line[:matched.start()] + if prefix.endswith('std::') or not prefix.endswith('::'): + required[''] = (linenum, 'string') + + for pattern, template, header in _re_pattern_algorithm_header: + if pattern.search(line): + required[header] = (linenum, template) + + # The following function is just a speed up, no semantics are changed. + if not '<' in line: # Reduces the cpu time usage by skipping lines. + continue + + for pattern, template, header in _re_pattern_templates: + if pattern.search(line): + required[header] = (linenum, template) + + # The policy is that if you #include something in foo.h you don't need to + # include it again in foo.cc. Here, we will look at possible includes. + # Let's flatten the include_state include_list and copy it into a dictionary. + include_dict = dict( + [item for sublist in include_state.include_list for item in sublist]) + + # Did we find the header for this file (if any) and successfully load it? + header_found = False + + # Use the absolute path so that matching works properly. + abs_filename = FileInfo(filename).FullName() + + # For Emacs's flymake. + # If cpplint is invoked from Emacs's flymake, a temporary file is generated + # by flymake and that file name might end with '_flymake.cc'. In that case, + # restore original file name here so that the corresponding header file can be + # found. + # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' + # instead of 'foo_flymake.h' + abs_filename = re.sub(r'_flymake\.cc$', '.cc', abs_filename) + + # include_dict is modified during iteration, so we iterate over a copy of + # the keys. + header_keys = include_dict.keys() + for header in header_keys: + (same_module, common_path) = FilesBelongToSameModule(abs_filename, + header) + fullpath = common_path + header + if same_module and UpdateIncludeState(fullpath, include_dict, io): + header_found = True + + # If we can't find the header file for a .cc, assume it's because we don't + # know where to look. In that case we'll give up as we're not sure they + # didn't include it in the .h file. + # TODO(unknown): Do a better job of finding .h files so we are confident that + # not having the .h file means there isn't one. + if filename.endswith('.cc') and not header_found: + return + + # All the lines have been processed, report the errors found. + for required_header_unstripped in required: + template = required[required_header_unstripped][1] + if required_header_unstripped.strip('<>"') not in include_dict: + error(filename, required[required_header_unstripped][0], + 'build/include_what_you_use', 4, 'Add #include ' + + required_header_unstripped + ' for ' + template) _RE_PATTERN_EXPLICIT_MAKEPAIR = re.compile(r'\bmake_pair\s*<') def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): - """Check that make_pair's template arguments are deduced. + """Check that make_pair's template arguments are deduced. G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are specified explicitly, and such use isn't intended in any case. @@ -5719,17 +5783,20 @@ def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) - if match: - error(filename, linenum, 'build/explicit_make_pair', - 4, # 4 = high confidence - 'For C++11-compatibility, omit template arguments from make_pair' - ' OR use pair directly OR if appropriate, construct a pair directly') + line = clean_lines.elided[linenum] + match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line) + if match: + error( + filename, + linenum, + 'build/explicit_make_pair', + 4, # 4 = high confidence + 'For C++11-compatibility, omit template arguments from make_pair' + ' OR use pair directly OR if appropriate, construct a pair directly') def CheckDefaultLambdaCaptures(filename, clean_lines, linenum, error): - """Check that default lambda captures are not used. + """Check that default lambda captures are not used. Args: filename: The name of the current file. @@ -5737,24 +5804,28 @@ def CheckDefaultLambdaCaptures(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # A lambda introducer specifies a default capture if it starts with "[=" - # or if it starts with "[&" _not_ followed by an identifier. - match = Match(r'^(.*)\[\s*(?:=|&[^\w])', line) - if match: - # Found a potential error, check what comes after the lambda-introducer. - # If it's not open parenthesis (for lambda-declarator) or open brace - # (for compound-statement), it's not a lambda. - line, _, pos = CloseExpression(clean_lines, linenum, len(match.group(1))) - if pos >= 0 and Match(r'^\s*[{(]', line[pos:]): - error(filename, linenum, 'build/c++11', - 4, # 4 = high confidence - 'Default lambda captures are an unapproved C++ feature.') + line = clean_lines.elided[linenum] + + # A lambda introducer specifies a default capture if it starts with "[=" + # or if it starts with "[&" _not_ followed by an identifier. + match = Match(r'^(.*)\[\s*(?:=|&[^\w])', line) + if match: + # Found a potential error, check what comes after the lambda-introducer. + # If it's not open parenthesis (for lambda-declarator) or open brace + # (for compound-statement), it's not a lambda. + line, _, pos = CloseExpression(clean_lines, linenum, + len(match.group(1))) + if pos >= 0 and Match(r'^\s*[{(]', line[pos:]): + error( + filename, + linenum, + 'build/c++11', + 4, # 4 = high confidence + 'Default lambda captures are an unapproved C++ feature.') def CheckRedundantVirtual(filename, clean_lines, linenum, error): - """Check if line contains a redundant "virtual" function-specifier. + """Check if line contains a redundant "virtual" function-specifier. Args: filename: The name of the current file. @@ -5762,63 +5833,64 @@ def CheckRedundantVirtual(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Look for "virtual" on current line. - line = clean_lines.elided[linenum] - virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) - if not virtual: return - - # Ignore "virtual" keywords that are near access-specifiers. These - # are only used in class base-specifier and do not apply to member - # functions. - if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or - Match(r'^\s+(public|protected|private)\b', virtual.group(3))): - return - - # Ignore the "virtual" keyword from virtual base classes. Usually - # there is a column on the same line in these cases (virtual base - # classes are rare in google3 because multiple inheritance is rare). - if Match(r'^.*[^:]:[^:].*$', line): return - - # Look for the next opening parenthesis. This is the start of the - # parameter list (possibly on the next line shortly after virtual). - # TODO(unknown): doesn't work if there are virtual functions with - # decltype() or other things that use parentheses, but csearch suggests - # that this is rare. - end_col = -1 - end_line = -1 - start_col = len(virtual.group(2)) - for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): - line = clean_lines.elided[start_line][start_col:] - parameter_list = Match(r'^([^(]*)\(', line) - if parameter_list: - # Match parentheses to find the end of the parameter list - (_, end_line, end_col) = CloseExpression( - clean_lines, start_line, start_col + len(parameter_list.group(1))) - break - start_col = 0 - - if end_col < 0: - return # Couldn't find end of parameter list, give up - - # Look for "override" or "final" after the parameter list - # (possibly on the next few lines). - for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): - line = clean_lines.elided[i][end_col:] - match = Search(r'\b(override|final)\b', line) - if match: - error(filename, linenum, 'readability/inheritance', 4, - ('"virtual" is redundant since function is ' - 'already declared as "%s"' % match.group(1))) + # Look for "virtual" on current line. + line = clean_lines.elided[linenum] + virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line) + if not virtual: return + + # Ignore "virtual" keywords that are near access-specifiers. These + # are only used in class base-specifier and do not apply to member + # functions. + if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or + Match(r'^\s+(public|protected|private)\b', virtual.group(3))): + return - # Set end_col to check whole lines after we are done with the - # first line. - end_col = 0 - if Search(r'[^\w]\s*$', line): - break + # Ignore the "virtual" keyword from virtual base classes. Usually + # there is a column on the same line in these cases (virtual base + # classes are rare in google3 because multiple inheritance is rare). + if Match(r'^.*[^:]:[^:].*$', line): return + + # Look for the next opening parenthesis. This is the start of the + # parameter list (possibly on the next line shortly after virtual). + # TODO(unknown): doesn't work if there are virtual functions with + # decltype() or other things that use parentheses, but csearch suggests + # that this is rare. + end_col = -1 + end_line = -1 + start_col = len(virtual.group(2)) + for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())): + line = clean_lines.elided[start_line][start_col:] + parameter_list = Match(r'^([^(]*)\(', line) + if parameter_list: + # Match parentheses to find the end of the parameter list + (_, end_line, end_col) = CloseExpression( + clean_lines, start_line, + start_col + len(parameter_list.group(1))) + break + start_col = 0 + + if end_col < 0: + return # Couldn't find end of parameter list, give up + + # Look for "override" or "final" after the parameter list + # (possibly on the next few lines). + for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())): + line = clean_lines.elided[i][end_col:] + match = Search(r'\b(override|final)\b', line) + if match: + error(filename, linenum, 'readability/inheritance', 4, + ('"virtual" is redundant since function is ' + 'already declared as "%s"' % match.group(1))) + + # Set end_col to check whole lines after we are done with the + # first line. + end_col = 0 + if Search(r'[^\w]\s*$', line): + break def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): - """Check if line contains a redundant "override" or "final" virt-specifier. + """Check if line contains a redundant "override" or "final" virt-specifier. Args: filename: The name of the current file. @@ -5826,32 +5898,30 @@ def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - # Look for closing parenthesis nearby. We need one to confirm where - # the declarator ends and where the virt-specifier starts to avoid - # false positives. - line = clean_lines.elided[linenum] - declarator_end = line.rfind(')') - if declarator_end >= 0: - fragment = line[declarator_end:] - else: - if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: - fragment = line + # Look for closing parenthesis nearby. We need one to confirm where + # the declarator ends and where the virt-specifier starts to avoid + # false positives. + line = clean_lines.elided[linenum] + declarator_end = line.rfind(')') + if declarator_end >= 0: + fragment = line[declarator_end:] else: - return - - # Check that at most one of "override" or "final" is present, not both - if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): - error(filename, linenum, 'readability/inheritance', 4, - ('"override" is redundant since function is ' - 'already declared as "final"')) - + if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0: + fragment = line + else: + return + # Check that at most one of "override" or "final" is present, not both + if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment): + error(filename, linenum, 'readability/inheritance', 4, + ('"override" is redundant since function is ' + 'already declared as "final"')) # Returns true if we are at a new block, and it is directly # inside of a namespace. def IsBlockInNameSpace(nesting_state, is_forward_declaration): - """Checks that the new block is directly in a namespace. + """Checks that the new block is directly in a namespace. Args: nesting_state: The _NestingState object that contains info about our state. @@ -5859,21 +5929,21 @@ def IsBlockInNameSpace(nesting_state, is_forward_declaration): Returns: Whether or not the new block is directly in a namespace. """ - if is_forward_declaration: - if len(nesting_state.stack) >= 1 and ( - isinstance(nesting_state.stack[-1], _NamespaceInfo)): - return True - else: - return False + if is_forward_declaration: + if len(nesting_state.stack) >= 1 and ( + isinstance(nesting_state.stack[-1], _NamespaceInfo)): + return True + else: + return False - return (len(nesting_state.stack) > 1 and - nesting_state.stack[-1].check_namespace_indentation and - isinstance(nesting_state.stack[-2], _NamespaceInfo)) + return (len(nesting_state.stack) > 1 and + nesting_state.stack[-1].check_namespace_indentation and + isinstance(nesting_state.stack[-2], _NamespaceInfo)) def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, raw_lines_no_comments, linenum): - """This method determines if we should apply our namespace indentation check. + """This method determines if we should apply our namespace indentation check. Args: nesting_state: The current nesting state. @@ -5888,17 +5958,17 @@ def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, only works for classes and namespaces inside of a namespace. """ - is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, - linenum) + is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments, + linenum) - if not (is_namespace_indent_item or is_forward_declaration): - return False + if not (is_namespace_indent_item or is_forward_declaration): + return False - # If we are in a macro, we do not want to check the namespace indentation. - if IsMacroDefinition(raw_lines_no_comments, linenum): - return False + # If we are in a macro, we do not want to check the namespace indentation. + if IsMacroDefinition(raw_lines_no_comments, linenum): + return False - return IsBlockInNameSpace(nesting_state, is_forward_declaration) + return IsBlockInNameSpace(nesting_state, is_forward_declaration) # Call this method if the line is directly inside of a namespace. @@ -5906,16 +5976,22 @@ def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item, # an inner namespace, it cannot be indented. def CheckItemIndentationInNamespace(filename, raw_lines_no_comments, linenum, error): - line = raw_lines_no_comments[linenum] - if Match(r'^\s+', line): - error(filename, linenum, 'runtime/indentation_namespace', 4, - 'Do not indent within a namespace') - - -def ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, + line = raw_lines_no_comments[linenum] + if Match(r'^\s+', line): + error(filename, linenum, 'runtime/indentation_namespace', 4, + 'Do not indent within a namespace') + + +def ProcessLine(filename, + file_extension, + clean_lines, + line, + include_state, + function_state, + nesting_state, + error, extra_check_functions=[]): - """Processes a single line in the file. + """Processes a single line in the file. Args: filename: Filename of the file that is being processed. @@ -5933,32 +6009,34 @@ def ProcessLine(filename, file_extension, clean_lines, line, run on each source line. Each function takes 4 arguments: filename, clean_lines, line, error """ - raw_lines = clean_lines.raw_lines - ParseNolintSuppressions(filename, raw_lines[line], line, error) - nesting_state.Update(filename, clean_lines, line, error) - CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, - error) - if nesting_state.InAsmBlock(): return - CheckForFunctionLengths(filename, clean_lines, line, function_state, error) - CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) - CheckStyle(filename, clean_lines, line, file_extension, nesting_state, error) - CheckLanguage(filename, clean_lines, line, file_extension, include_state, - nesting_state, error) - CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) - CheckForNonStandardConstructs(filename, clean_lines, line, - nesting_state, error) - CheckVlogArguments(filename, clean_lines, line, error) - CheckPosixThreading(filename, clean_lines, line, error) - CheckInvalidIncrement(filename, clean_lines, line, error) - CheckMakePairUsesDeduction(filename, clean_lines, line, error) - CheckDefaultLambdaCaptures(filename, clean_lines, line, error) - CheckRedundantVirtual(filename, clean_lines, line, error) - CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) - for check_fn in extra_check_functions: - check_fn(filename, clean_lines, line, error) + raw_lines = clean_lines.raw_lines + ParseNolintSuppressions(filename, raw_lines[line], line, error) + nesting_state.Update(filename, clean_lines, line, error) + CheckForNamespaceIndentation(filename, nesting_state, clean_lines, line, + error) + if nesting_state.InAsmBlock(): return + CheckForFunctionLengths(filename, clean_lines, line, function_state, error) + CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) + CheckStyle(filename, clean_lines, line, file_extension, nesting_state, + error) + CheckLanguage(filename, clean_lines, line, file_extension, include_state, + nesting_state, error) + CheckForNonConstReference(filename, clean_lines, line, nesting_state, error) + CheckForNonStandardConstructs(filename, clean_lines, line, nesting_state, + error) + CheckVlogArguments(filename, clean_lines, line, error) + CheckPosixThreading(filename, clean_lines, line, error) + CheckInvalidIncrement(filename, clean_lines, line, error) + CheckMakePairUsesDeduction(filename, clean_lines, line, error) + CheckDefaultLambdaCaptures(filename, clean_lines, line, error) + CheckRedundantVirtual(filename, clean_lines, line, error) + CheckRedundantOverrideOrFinal(filename, clean_lines, line, error) + for check_fn in extra_check_functions: + check_fn(filename, clean_lines, line, error) + def FlagCxx11Features(filename, clean_lines, linenum, error): - """Flag those c++11 features that we only allow in certain places. + """Flag those c++11 features that we only allow in certain places. Args: filename: The name of the current file. @@ -5966,46 +6044,48 @@ def FlagCxx11Features(filename, clean_lines, linenum, error): linenum: The number of the line to check. error: The function to call with any errors found. """ - line = clean_lines.elided[linenum] - - # Flag unapproved C++11 headers. - include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) - if include and include.group(1) in ('cfenv', - 'condition_variable', - 'fenv.h', - 'future', - 'mutex', - 'thread', - 'chrono', - 'ratio', - 'regex', - 'system_error', - ): - error(filename, linenum, 'build/c++11', 5, - ('<%s> is an unapproved C++11 header.') % include.group(1)) - - # The only place where we need to worry about C++11 keywords and library - # features in preprocessor directives is in macro definitions. - if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return - - # These are classes and free functions. The classes are always - # mentioned as std::*, but we only catch the free functions if - # they're not found by ADL. They're alphabetical by header. - for top_name in ( - # type_traits - 'alignment_of', - 'aligned_union', - ): - if Search(r'\bstd::%s\b' % top_name, line): - error(filename, linenum, 'build/c++11', 5, - ('std::%s is an unapproved C++11 class or function. Send c-style ' - 'an example of where it would make your code more readable, and ' - 'they may let you use it.') % top_name) - - -def ProcessFileData(filename, file_extension, lines, error, + line = clean_lines.elided[linenum] + + # Flag unapproved C++11 headers. + include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line) + if include and include.group(1) in ( + 'cfenv', + 'condition_variable', + 'fenv.h', + 'future', + 'mutex', + 'thread', + 'chrono', + 'ratio', + 'regex', + 'system_error', ): + error(filename, linenum, 'build/c++11', 5, + ('<%s> is an unapproved C++11 header.') % include.group(1)) + + # The only place where we need to worry about C++11 keywords and library + # features in preprocessor directives is in macro definitions. + if Match(r'\s*#', line) and not Match(r'\s*#\s*define\b', line): return + + # These are classes and free functions. The classes are always + # mentioned as std::*, but we only catch the free functions if + # they're not found by ADL. They're alphabetical by header. + for top_name in ( + # type_traits + 'alignment_of', + 'aligned_union', ): + if Search(r'\bstd::%s\b' % top_name, line): + error(filename, linenum, 'build/c++11', 5, ( + 'std::%s is an unapproved C++11 class or function. Send c-style ' + 'an example of where it would make your code more readable, and ' + 'they may let you use it.') % top_name) + + +def ProcessFileData(filename, + file_extension, + lines, + error, extra_check_functions=[]): - """Performs lint checks and reports any errors to the given error function. + """Performs lint checks and reports any errors to the given error function. Args: filename: Filename of the file that is being processed. @@ -6018,44 +6098,44 @@ def ProcessFileData(filename, file_extension, lines, error, run on each source line. Each function takes 4 arguments: filename, clean_lines, line, error """ - lines = (['// marker so line numbers and indices both start at 1'] + lines + - ['// marker so line numbers end in a known way']) + lines = (['// marker so line numbers and indices both start at 1'] + lines + + ['// marker so line numbers end in a known way']) - include_state = _IncludeState() - function_state = _FunctionState() - nesting_state = NestingState() + include_state = _IncludeState() + function_state = _FunctionState() + nesting_state = NestingState() - ResetNolintSuppressions() + ResetNolintSuppressions() - CheckForCopyright(filename, lines, error) + CheckForCopyright(filename, lines, error) - RemoveMultiLineComments(filename, lines, error) - clean_lines = CleansedLines(lines) + RemoveMultiLineComments(filename, lines, error) + clean_lines = CleansedLines(lines) - if file_extension == 'h': - CheckForHeaderGuard(filename, clean_lines, error) + if file_extension == 'h': + CheckForHeaderGuard(filename, clean_lines, error) - for line in xrange(clean_lines.NumLines()): - ProcessLine(filename, file_extension, clean_lines, line, - include_state, function_state, nesting_state, error, - extra_check_functions) - FlagCxx11Features(filename, clean_lines, line, error) - nesting_state.CheckCompletedBlocks(filename, error) + for line in xrange(clean_lines.NumLines()): + ProcessLine(filename, file_extension, clean_lines, line, include_state, + function_state, nesting_state, error, extra_check_functions) + FlagCxx11Features(filename, clean_lines, line, error) + nesting_state.CheckCompletedBlocks(filename, error) - CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) - - # Check that the .cc file has included its header if it exists. - if file_extension == 'cc': - CheckHeaderFileIncluded(filename, include_state, error) + CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) - # We check here rather than inside ProcessLine so that we see raw - # lines rather than "cleaned" lines. - CheckForBadCharacters(filename, lines, error) + # Check that the .cc file has included its header if it exists. + if file_extension == 'cc': + CheckHeaderFileIncluded(filename, include_state, error) + + # We check here rather than inside ProcessLine so that we see raw + # lines rather than "cleaned" lines. + CheckForBadCharacters(filename, lines, error) + + CheckForNewlineAtEOF(filename, lines, error) - CheckForNewlineAtEOF(filename, lines, error) def ProcessConfigOverrides(filename): - """ Loads the configuration files and processes the config overrides. + """ Loads the configuration files and processes the config overrides. Args: filename: The name of the file being processed by the linter. @@ -6064,74 +6144,76 @@ def ProcessConfigOverrides(filename): False if the current |filename| should not be processed further. """ - abs_filename = os.path.abspath(filename) - cfg_filters = [] - keep_looking = True - while keep_looking: - abs_path, base_name = os.path.split(abs_filename) - if not base_name: - break # Reached the root directory. - - cfg_file = os.path.join(abs_path, "CPPLINT.cfg") - abs_filename = abs_path - if not os.path.isfile(cfg_file): - continue - - try: - with open(cfg_file) as file_handle: - for line in file_handle: - line, _, _ = line.partition('#') # Remove comments. - if not line.strip(): + abs_filename = os.path.abspath(filename) + cfg_filters = [] + keep_looking = True + while keep_looking: + abs_path, base_name = os.path.split(abs_filename) + if not base_name: + break # Reached the root directory. + + cfg_file = os.path.join(abs_path, "CPPLINT.cfg") + abs_filename = abs_path + if not os.path.isfile(cfg_file): continue - name, _, val = line.partition('=') - name = name.strip() - val = val.strip() - if name == 'set noparent': - keep_looking = False - elif name == 'filter': - cfg_filters.append(val) - elif name == 'exclude_files': - # When matching exclude_files pattern, use the base_name of - # the current file name or the directory name we are processing. - # For example, if we are checking for lint errors in /foo/bar/baz.cc - # and we found the .cfg file at /foo/CPPLINT.cfg, then the config - # file's "exclude_files" filter is meant to be checked against "bar" - # and not "baz" nor "bar/baz.cc". - if base_name: - pattern = re.compile(val) - if pattern.match(base_name): - sys.stderr.write('Ignoring "%s": file excluded by "%s". ' - 'File path component "%s" matches ' - 'pattern "%s"\n' % - (filename, cfg_file, base_name, val)) - return False - elif name == 'linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - sys.stderr.write('Line length must be numeric.') - else: + try: + with open(cfg_file) as file_handle: + for line in file_handle: + line, _, _ = line.partition('#') # Remove comments. + if not line.strip(): + continue + + name, _, val = line.partition('=') + name = name.strip() + val = val.strip() + if name == 'set noparent': + keep_looking = False + elif name == 'filter': + cfg_filters.append(val) + elif name == 'exclude_files': + # When matching exclude_files pattern, use the base_name of + # the current file name or the directory name we are processing. + # For example, if we are checking for lint errors in /foo/bar/baz.cc + # and we found the .cfg file at /foo/CPPLINT.cfg, then the config + # file's "exclude_files" filter is meant to be checked against "bar" + # and not "baz" nor "bar/baz.cc". + if base_name: + pattern = re.compile(val) + if pattern.match(base_name): + sys.stderr.write( + 'Ignoring "%s": file excluded by "%s". ' + 'File path component "%s" matches ' + 'pattern "%s"\n' % + (filename, cfg_file, base_name, val)) + return False + elif name == 'linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + sys.stderr.write('Line length must be numeric.') + else: + sys.stderr.write( + 'Invalid configuration option (%s) in file %s\n' % + (name, cfg_file)) + + except IOError: sys.stderr.write( - 'Invalid configuration option (%s) in file %s\n' % - (name, cfg_file)) - - except IOError: - sys.stderr.write( - "Skipping config file '%s': Can't open for reading\n" % cfg_file) - keep_looking = False + "Skipping config file '%s': Can't open for reading\n" % + cfg_file) + keep_looking = False - # Apply all the accumulated filters in reverse order (top-level directory - # config options having the least priority). - for filter in reversed(cfg_filters): - _AddFilters(filter) + # Apply all the accumulated filters in reverse order (top-level directory + # config options having the least priority). + for filter in reversed(cfg_filters): + _AddFilters(filter) - return True + return True def ProcessFile(filename, vlevel, extra_check_functions=[]): - """Does google-lint on a single file. + """Does google-lint on a single file. Args: filename: The name of the file to parse. @@ -6144,104 +6226,105 @@ def ProcessFile(filename, vlevel, extra_check_functions=[]): arguments: filename, clean_lines, line, error """ - _SetVerboseLevel(vlevel) - _BackupFilters() + _SetVerboseLevel(vlevel) + _BackupFilters() - if not ProcessConfigOverrides(filename): - _RestoreFilters() - return - - lf_lines = [] - crlf_lines = [] - try: - # Support the UNIX convention of using "-" for stdin. Note that - # we are not opening the file with universal newline support - # (which codecs doesn't support anyway), so the resulting lines do - # contain trailing '\r' characters if we are reading a file that - # has CRLF endings. - # If after the split a trailing '\r' is present, it is removed - # below. - if filename == '-': - lines = codecs.StreamReaderWriter(sys.stdin, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace').read().split('\n') + if not ProcessConfigOverrides(filename): + _RestoreFilters() + return + + lf_lines = [] + crlf_lines = [] + try: + # Support the UNIX convention of using "-" for stdin. Note that + # we are not opening the file with universal newline support + # (which codecs doesn't support anyway), so the resulting lines do + # contain trailing '\r' characters if we are reading a file that + # has CRLF endings. + # If after the split a trailing '\r' is present, it is removed + # below. + if filename == '-': + lines = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace').read().split('\n') + else: + lines = codecs.open(filename, 'r', 'utf8', + 'replace').read().split('\n') + + # Remove trailing '\r'. + # The -1 accounts for the extra trailing blank line we get from split() + for linenum in range(len(lines) - 1): + if lines[linenum].endswith('\r'): + lines[linenum] = lines[linenum].rstrip('\r') + crlf_lines.append(linenum + 1) + else: + lf_lines.append(linenum + 1) + + except IOError: + sys.stderr.write("Skipping input '%s': Can't open for reading\n" % + filename) + _RestoreFilters() + return + + # Note, if no dot is found, this will give the entire filename as the ext. + file_extension = filename[filename.rfind('.') + 1:] + + # When reading from stdin, the extension is unknown, so no cpplint tests + # should rely on the extension. + if filename != '-' and file_extension not in _valid_extensions: + sys.stderr.write('Ignoring %s; not a valid file name ' + '(%s)\n' % (filename, ', '.join(_valid_extensions))) else: - lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') - - # Remove trailing '\r'. - # The -1 accounts for the extra trailing blank line we get from split() - for linenum in range(len(lines) - 1): - if lines[linenum].endswith('\r'): - lines[linenum] = lines[linenum].rstrip('\r') - crlf_lines.append(linenum + 1) - else: - lf_lines.append(linenum + 1) - - except IOError: - sys.stderr.write( - "Skipping input '%s': Can't open for reading\n" % filename) - _RestoreFilters() - return - - # Note, if no dot is found, this will give the entire filename as the ext. - file_extension = filename[filename.rfind('.') + 1:] - - # When reading from stdin, the extension is unknown, so no cpplint tests - # should rely on the extension. - if filename != '-' and file_extension not in _valid_extensions: - sys.stderr.write('Ignoring %s; not a valid file name ' - '(%s)\n' % (filename, ', '.join(_valid_extensions))) - else: - ProcessFileData(filename, file_extension, lines, Error, - extra_check_functions) - - # If end-of-line sequences are a mix of LF and CR-LF, issue - # warnings on the lines with CR. - # - # Don't issue any warnings if all lines are uniformly LF or CR-LF, - # since critique can handle these just fine, and the style guide - # doesn't dictate a particular end of line sequence. - # - # We can't depend on os.linesep to determine what the desired - # end-of-line sequence should be, since that will return the - # server-side end-of-line sequence. - if lf_lines and crlf_lines: - # Warn on every line with CR. An alternative approach might be to - # check whether the file is mostly CRLF or just LF, and warn on the - # minority, we bias toward LF here since most tools prefer LF. - for linenum in crlf_lines: - Error(filename, linenum, 'whitespace/newline', 1, - 'Unexpected \\r (^M) found; better to use only \\n') + ProcessFileData(filename, file_extension, lines, Error, + extra_check_functions) - sys.stdout.write('Done processing %s\n' % filename) - _RestoreFilters() + # If end-of-line sequences are a mix of LF and CR-LF, issue + # warnings on the lines with CR. + # + # Don't issue any warnings if all lines are uniformly LF or CR-LF, + # since critique can handle these just fine, and the style guide + # doesn't dictate a particular end of line sequence. + # + # We can't depend on os.linesep to determine what the desired + # end-of-line sequence should be, since that will return the + # server-side end-of-line sequence. + if lf_lines and crlf_lines: + # Warn on every line with CR. An alternative approach might be to + # check whether the file is mostly CRLF or just LF, and warn on the + # minority, we bias toward LF here since most tools prefer LF. + for linenum in crlf_lines: + Error(filename, linenum, 'whitespace/newline', 1, + 'Unexpected \\r (^M) found; better to use only \\n') + + sys.stdout.write('Done processing %s\n' % filename) + _RestoreFilters() def PrintUsage(message): - """Prints a brief usage string and exits, optionally with an error message. + """Prints a brief usage string and exits, optionally with an error message. Args: message: The optional error message. """ - sys.stderr.write(_USAGE) - if message: - sys.exit('\nFATAL ERROR: ' + message) - else: - sys.exit(1) + sys.stderr.write(_USAGE) + if message: + sys.exit('\nFATAL ERROR: ' + message) + else: + sys.exit(1) def PrintCategories(): - """Prints a list of all the error-categories used by error messages. + """Prints a list of all the error-categories used by error messages. These are the categories used to filter messages via --filter. """ - sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) - sys.exit(0) + sys.stderr.write(''.join(' %s\n' % cat for cat in _ERROR_CATEGORIES)) + sys.exit(0) def ParseArguments(args): - """Parses the command line arguments. + """Parses the command line arguments. This may set the output format and verbosity level as side-effects. @@ -6251,82 +6334,82 @@ def ParseArguments(args): Returns: The list of filenames to lint. """ - try: - (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', - 'counting=', - 'filter=', - 'root=', - 'linelength=', - 'extensions=']) - except getopt.GetoptError: - PrintUsage('Invalid arguments.') - - verbosity = _VerboseLevel() - output_format = _OutputFormat() - filters = '' - counting_style = '' - - for (opt, val) in opts: - if opt == '--help': - PrintUsage(None) - elif opt == '--output': - if val not in ('emacs', 'vs7', 'eclipse'): - PrintUsage('The only allowed output formats are emacs, vs7 and eclipse.') - output_format = val - elif opt == '--verbose': - verbosity = int(val) - elif opt == '--filter': - filters = val - if not filters: - PrintCategories() - elif opt == '--counting': - if val not in ('total', 'toplevel', 'detailed'): - PrintUsage('Valid counting options are total, toplevel, and detailed') - counting_style = val - elif opt == '--root': - global _root - _root = val - elif opt == '--linelength': - global _line_length - try: - _line_length = int(val) - except ValueError: - PrintUsage('Line length must be digits.') - elif opt == '--extensions': - global _valid_extensions - try: - _valid_extensions = set(val.split(',')) - except ValueError: - PrintUsage('Extensions must be comma seperated list.') - - if not filenames: - PrintUsage('No files were specified.') - - _SetOutputFormat(output_format) - _SetVerboseLevel(verbosity) - _SetFilters(filters) - _SetCountingStyle(counting_style) - - return filenames + try: + (opts, filenames) = getopt.getopt(args, '', [ + 'help', 'output=', 'verbose=', 'counting=', 'filter=', 'root=', + 'linelength=', 'extensions=' + ]) + except getopt.GetoptError: + PrintUsage('Invalid arguments.') + + verbosity = _VerboseLevel() + output_format = _OutputFormat() + filters = '' + counting_style = '' + + for (opt, val) in opts: + if opt == '--help': + PrintUsage(None) + elif opt == '--output': + if val not in ('emacs', 'vs7', 'eclipse'): + PrintUsage( + 'The only allowed output formats are emacs, vs7 and eclipse.' + ) + output_format = val + elif opt == '--verbose': + verbosity = int(val) + elif opt == '--filter': + filters = val + if not filters: + PrintCategories() + elif opt == '--counting': + if val not in ('total', 'toplevel', 'detailed'): + PrintUsage( + 'Valid counting options are total, toplevel, and detailed') + counting_style = val + elif opt == '--root': + global _root + _root = val + elif opt == '--linelength': + global _line_length + try: + _line_length = int(val) + except ValueError: + PrintUsage('Line length must be digits.') + elif opt == '--extensions': + global _valid_extensions + try: + _valid_extensions = set(val.split(',')) + except ValueError: + PrintUsage('Extensions must be comma seperated list.') + + if not filenames: + PrintUsage('No files were specified.') + + _SetOutputFormat(output_format) + _SetVerboseLevel(verbosity) + _SetFilters(filters) + _SetCountingStyle(counting_style) + + return filenames def main(): - filenames = ParseArguments(sys.argv[1:]) + filenames = ParseArguments(sys.argv[1:]) - # Change stderr to write with replacement characters so we don't die - # if we try to print something containing non-ASCII characters. - sys.stderr = codecs.StreamReaderWriter(sys.stderr, - codecs.getreader('utf8'), - codecs.getwriter('utf8'), - 'replace') + # Change stderr to write with replacement characters so we don't die + # if we try to print something containing non-ASCII characters. + sys.stderr = codecs.StreamReaderWriter(sys.stderr, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), 'replace') - _cpplint_state.ResetErrorCounts() - for filename in filenames: - ProcessFile(filename, _cpplint_state.verbose_level) - _cpplint_state.PrintErrorCounts() + _cpplint_state.ResetErrorCounts() + for filename in filenames: + ProcessFile(filename, _cpplint_state.verbose_level) + _cpplint_state.PrintErrorCounts() - sys.exit(_cpplint_state.error_count > 0) + sys.exit(_cpplint_state.error_count > 0) if __name__ == '__main__': - main() + main() diff --git a/paddle/scripts/deb/build_scripts/build.sh b/paddle/scripts/deb/build_scripts/build.sh index 662d2a9103f7da62d96650f490688d02b2c4669e..66a1cfb247dad0292c0832046fb121d14b15b5ba 100755 --- a/paddle/scripts/deb/build_scripts/build.sh +++ b/paddle/scripts/deb/build_scripts/build.sh @@ -33,5 +33,3 @@ cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ make -j `nproc` cpack -D CPACK_GENERATOR='DEB' .. mv *.deb ~/dist/gpu-noavx - - diff --git a/paddle/scripts/docker/Dockerfile.cpu b/paddle/scripts/docker/Dockerfile.cpu index 3aa8cb1a3a8695d513ca0682c0ac2a269e6589da..a833c69c66900ee23176909ffce0835f6637c391 100644 --- a/paddle/scripts/docker/Dockerfile.cpu +++ b/paddle/scripts/docker/Dockerfile.cpu @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-demo b/paddle/scripts/docker/Dockerfile.cpu-demo index 22c0b9e701bfc06230d7fcb1f7b3c49e8e3d0d0f..1fda1e472b290c970a29e927db001abd949d9e62 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-demo +++ b/paddle/scripts/docker/Dockerfile.cpu-demo @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.cpu-devel b/paddle/scripts/docker/Dockerfile.cpu-devel index b40f3c0a30ba36571883a9881f6146e630cca187..66bdc978ddcb4bdd2c670cbbb4004bfaba54c8b3 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-devel +++ b/paddle/scripts/docker/Dockerfile.cpu-devel @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx b/paddle/scripts/docker/Dockerfile.cpu-noavx index 5cb5ac7dc4e6811ffdcacae7188dae078adc4030..d0ba30e55afb2c91875838cf8e59f51250ce6f3a 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo index bec401960efb2329a54e0e80043eba7d9ab36d9c..28439b4bdfab437ae947c1fba637632c149fcd1d 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel index b7c3eaed97aa5f11c80a9082fc7733ffd475e965..eb4739d6dc742407549cfdaa73544282c260db87 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu b/paddle/scripts/docker/Dockerfile.gpu index b7f5b6d93df50638fbb5e04c61aeed112371ef5b..fa61cfeec851f128f3f073afc64e0499b322e4dd 100644 --- a/paddle/scripts/docker/Dockerfile.gpu +++ b/paddle/scripts/docker/Dockerfile.gpu @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-demo b/paddle/scripts/docker/Dockerfile.gpu-demo index 2d1411de09f2ac37aa129f51d5d4e98baa22023b..4f5417c1af072b6c5366dc5fe0dbedae3f9c880e 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-demo +++ b/paddle/scripts/docker/Dockerfile.gpu-demo @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.gpu-devel b/paddle/scripts/docker/Dockerfile.gpu-devel index eb13f4304fa06203fcf319b6b1cf4adf087cb044..37cfced1908861b9131c1dd80a610eadb9bcd882 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-devel +++ b/paddle/scripts/docker/Dockerfile.gpu-devel @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx b/paddle/scripts/docker/Dockerfile.gpu-noavx index 0944b0e152af3ba3e4794091c6a76e05b888573b..95fb125b799e8f0403cdae0d2c191188a52285e6 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo index 2da2a55d696a38016b0eb13cd263977026cb1f2b..b5fbe4b941d6814cde304a116da253dd48ed41c8 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel index 9f551462f206aaf59fca7ee5bfe258f83cfdd0ca..531c8ec7ae30cd688b06fe1ba03bd215be81096c 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.m4 b/paddle/scripts/docker/Dockerfile.m4 index 129d21b36abd9443e62276c75da3eeb19fa246de..57c865584413381abc54dc759405360a5ae354f3 100644 --- a/paddle/scripts/docker/Dockerfile.m4 +++ b/paddle/scripts/docker/Dockerfile.m4 @@ -1,6 +1,7 @@ FROM PADDLE_BASE_IMAGE MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=PADDLE_WITH_GPU ENV IS_DEVEL=PADDLE_IS_DEVEL ENV WITH_DEMO=PADDLE_WITH_DEMO diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 33689e736cda76ea5731d847260b399fc8c6e484..ec5f3bd967d3569ee058a2e12d85fc50ba25c69d 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -23,6 +23,7 @@ fi cd ~ git clone https://github.com/baidu/Paddle.git paddle cd paddle +git checkout ${GIT_CHECKOUT} mkdir build cd build cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ diff --git a/paddle/scripts/docker/generate.sh b/paddle/scripts/docker/generate.sh index 8a50aefd34955fcc54c25c833154997472449f93..2ad7527db127f3bd2018a7a1f5b40dacfecca6da 100644 --- a/paddle/scripts/docker/generate.sh +++ b/paddle/scripts/docker/generate.sh @@ -58,4 +58,3 @@ m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ -DPADDLE_WITH_AVX=OFF \ Dockerfile.m4 > Dockerfile.gpu-noavx-demo - diff --git a/paddle/scripts/travis/common.sh b/paddle/scripts/travis/common.sh index 37e27d665b12fb5b3b8ec7ad245d4587cb0361d6..9b6e420ca7931f0d17da461c7579bf4dc69e18e0 100755 --- a/paddle/scripts/travis/common.sh +++ b/paddle/scripts/travis/common.sh @@ -2,4 +2,3 @@ set -e mkdir -p ../../../build cd ../../../build - diff --git a/paddle/trainer/tests/__init__.py b/paddle/trainer/tests/__init__.py index 7f9e87eee6037666b86420fba194624859d356b3..c90af2ee000d46a032984ee23559e7e99b49ddad 100644 --- a/paddle/trainer/tests/__init__.py +++ b/paddle/trainer/tests/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/trainer/tests/config_parser_test.py b/paddle/trainer/tests/config_parser_test.py index 5ca874cec7914a20f79c2c7b1873c5bd04f60dca..c5ec315d6b01b0a5a3f73673e1756e9c06d685ba 100644 --- a/paddle/trainer/tests/config_parser_test.py +++ b/paddle/trainer/tests/config_parser_test.py @@ -17,6 +17,6 @@ from paddle.trainer.config_parser import parse_config_and_serialize if __name__ == '__main__': parse_config_and_serialize('trainer/tests/test_config.conf', '') parse_config_and_serialize( - 'trainer/tests/sample_trainer_config.conf', + 'trainer/tests/sample_trainer_config.conf', 'extension_module_name=paddle.trainer.config_parser_extension') parse_config_and_serialize('gserver/tests/pyDataProvider/trainer.conf', '') diff --git a/paddle/trainer/tests/gen_proto_data.py b/paddle/trainer/tests/gen_proto_data.py index c818a94bee7c28b0245d28dd62353d46444cb592..a3dbc10c886e183582b44fee479d5ffb074193ef 100644 --- a/paddle/trainer/tests/gen_proto_data.py +++ b/paddle/trainer/tests/gen_proto_data.py @@ -21,8 +21,7 @@ import logging import pprint logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) @@ -36,33 +35,32 @@ num_original_columns = 3 # [[-1,0], [0,0]] means previous token at column 0 and current token at # column 0 are combined as one feature. patterns = [ - [[-2,0]], - [[-1,0]], - [[0,0]], - [[1,0]], - [[2,0]], - - [[-1,0], [0,0]], - [[0,0], [1,0]], - - [[-2,1]], - [[-1,1]], - [[0,1]], - [[1,1]], - [[2,1]], - [[-2,1], [-1,1]], - [[-1,1], [0,1]], - [[0,1], [1,1]], - [[1,1], [2,1]], - - [[-2,1], [-1,1], [0,1]], - [[-1,1], [0,1], [1,1]], - [[0,1], [1,1], [2,1]], + [[-2, 0]], + [[-1, 0]], + [[0, 0]], + [[1, 0]], + [[2, 0]], + [[-1, 0], [0, 0]], + [[0, 0], [1, 0]], + [[-2, 1]], + [[-1, 1]], + [[0, 1]], + [[1, 1]], + [[2, 1]], + [[-2, 1], [-1, 1]], + [[-1, 1], [0, 1]], + [[0, 1], [1, 1]], + [[1, 1], [2, 1]], + [[-2, 1], [-1, 1], [0, 1]], + [[-1, 1], [0, 1], [1, 1]], + [[0, 1], [1, 1], [2, 1]], ] + def make_features(sequence): length = len(sequence) num_features = len(sequence[0]) + def get_features(pos): if pos < 0: return ['#B%s' % -pos] * num_features @@ -72,9 +70,10 @@ def make_features(sequence): for i in xrange(length): for pattern in patterns: - fname = '/'.join([get_features(i+pos)[f] for pos, f in pattern]) + fname = '/'.join([get_features(i + pos)[f] for pos, f in pattern]) sequence[i].append(fname) + ''' Source file format: Each line is for one timestep. The features are separated by space. @@ -87,6 +86,8 @@ i-th column. return a list of dict for each column ''' + + def create_dictionaries(filename, cutoff, oov_policy): def add_to_dict(sequence, dicts): num_features = len(dicts) @@ -118,7 +119,6 @@ def create_dictionaries(filename, cutoff, oov_policy): features = line.split(' ') sequence.append(features) - for i in xrange(num_features): dct = dicts[i] n = 1 if oov_policy[i] == OOV_POLICY_USE else 0 @@ -161,12 +161,9 @@ existed in dicts[i] will be assigned to id 0. if oov_policy[i] == OOV_POLICY_ERROR, all features in i-th column MUST exist in dicts[i]. ''' -def gen_proto_file( - input_file, - dicts, - oov_policy, - output_file): + +def gen_proto_file(input_file, dicts, oov_policy, output_file): def write_sequence(out, sequence): num_features = len(dicts) is_beginning = True @@ -213,8 +210,8 @@ def gen_proto_file( if patterns: slot_def = header.slot_defs.add() slot_def.type = DataFormat.SlotDef.VECTOR_SPARSE_NON_VALUE - slot_def.dim = sum([len(dicts[i]) - for i in xrange(num_original_columns, len(dicts))]) + slot_def.dim = sum( + [len(dicts[i]) for i in xrange(num_original_columns, len(dicts))]) logger.info("feature_dim=%s" % slot_def.dim) for i in xrange(num_original_columns): @@ -242,30 +239,31 @@ def gen_proto_file( logger.info("num_sequences=%s" % num_sequences) + dict2 = { - 'B-ADJP': 0, - 'I-ADJP': 1, - 'B-ADVP': 2, - 'I-ADVP': 3, - 'B-CONJP': 4, - 'I-CONJP': 5, - 'B-INTJ': 6, - 'I-INTJ': 7, - 'B-LST': 8, - 'I-LST': 9, - 'B-NP': 10, - 'I-NP': 11, - 'B-PP': 12, - 'I-PP': 13, - 'B-PRT': 14, - 'I-PRT': 15, - 'B-SBAR': 16, - 'I-SBAR': 17, - 'B-UCP': 18, - 'I-UCP': 19, - 'B-VP': 20, - 'I-VP': 21, - 'O': 22 + 'B-ADJP': 0, + 'I-ADJP': 1, + 'B-ADVP': 2, + 'I-ADVP': 3, + 'B-CONJP': 4, + 'I-CONJP': 5, + 'B-INTJ': 6, + 'I-INTJ': 7, + 'B-LST': 8, + 'I-LST': 9, + 'B-NP': 10, + 'I-NP': 11, + 'B-PP': 12, + 'I-PP': 13, + 'B-PRT': 14, + 'I-PRT': 15, + 'B-SBAR': 16, + 'I-SBAR': 17, + 'B-UCP': 18, + 'I-UCP': 19, + 'B-VP': 20, + 'I-VP': 21, + 'O': 22 } if __name__ == '__main__': @@ -273,16 +271,9 @@ if __name__ == '__main__': cutoff += [3] * len(patterns) oov_policy = [OOV_POLICY_IGNORE, OOV_POLICY_ERROR, OOV_POLICY_ERROR] oov_policy += [OOV_POLICY_IGNORE] * len(patterns) - dicts = create_dictionaries( - 'trainer/tests/train.txt', cutoff, oov_policy) + dicts = create_dictionaries('trainer/tests/train.txt', cutoff, oov_policy) dicts[2] = dict2 - gen_proto_file( - 'trainer/tests/train.txt', - dicts, - oov_policy, - 'trainer/tests/train_proto.bin') - gen_proto_file( - 'trainer/tests/test.txt', - dicts, - oov_policy, - 'trainer/tests/test_proto.bin') + gen_proto_file('trainer/tests/train.txt', dicts, oov_policy, + 'trainer/tests/train_proto.bin') + gen_proto_file('trainer/tests/test.txt', dicts, oov_policy, + 'trainer/tests/test_proto.bin') diff --git a/paddle/trainer/tests/test.txt b/paddle/trainer/tests/test.txt index 68e7f72e3d8d4ee4e592309cfc230fad24a810d4..3ad503b34f2e1a84c632d0894f180b5cf9ac550a 100644 --- a/paddle/trainer/tests/test.txt +++ b/paddle/trainer/tests/test.txt @@ -998,4 +998,3 @@ from IN B-PP Friday NNP B-NP 's POS B-NP Tokyo NNP I-NP - diff --git a/paddle/trainer/tests/testPyDataWrapper.py b/paddle/trainer/tests/testPyDataWrapper.py index 49bd760f4e20e2a12e5686b3193bdba2895612e4..4607bec24e1fec6f8b9996eb32fe991dbbe3ed79 100644 --- a/paddle/trainer/tests/testPyDataWrapper.py +++ b/paddle/trainer/tests/testPyDataWrapper.py @@ -21,7 +21,10 @@ import json import string -@provider(slots=[SparseNonValueSlot(10), DenseSlot(2), SparseValueSlot(10), StringSlot(1), IndexSlot(3)]) +@provider(slots=[ + SparseNonValueSlot(10), DenseSlot(2), SparseValueSlot(10), StringSlot(1), + IndexSlot(3) +]) def processNonSequenceData(obj, filename): with open(filename, "rb") as f: for line in f: @@ -50,6 +53,7 @@ val_randomer = lambda: random.uniform(-1.0, 1.0) seq_count_randomer = lambda: random.randrange(1, SEQUENCE_LIMIT) str_count_randomer = lambda: random.randrange(1, STRING_LIMIT) + class IDRandomer(): # A random generator, return unique id def __init__(self): self.id_set = set() @@ -61,38 +65,57 @@ class IDRandomer(): # A random generator, return unique id return idx else: return self.__call__() + + # SparseValueSlot def sparse_value_creator(_): rand = IDRandomer() return [(rand(), val_randomer()) for _ in xrange(sparse_count_randomer())] + + sparse_value = map(sparse_value_creator, range(seq_count_randomer())) + # DenseSlot def dense_creator(_): return [val_randomer() for _ in xrange(SPARSE_ID_LIMIT)] + + dense = map(dense_creator, range(seq_count_randomer())) + # SparseNonValueSlot def sparse_creator(_): rand = IDRandomer() return [rand() for _ in xrange(sparse_count_randomer())] + + sparse_nonvalue = map(sparse_creator, range(seq_count_randomer())) # IndexSlot ids = [sparse_id_randomer() for _ in range(seq_count_randomer())] + # StringSlot -def random_str(size = 8, chars=string.ascii_letters + string.digits): +def random_str(size=8, chars=string.ascii_letters + string.digits): return ''.join(random.choice(chars) for _ in range(size)) + + strs = [random_str(str_count_randomer()) for _ in range(seq_count_randomer())] + def processSeqAndGenerateDataInit(obj, *args, **kwargs): obj.json_filename = kwargs.get("load_data_args", "test_data.json") -@provider(slots=[SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), - SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), - StringSlot(SPARSE_ID_LIMIT)], - use_seq=True, init_hook=processSeqAndGenerateDataInit) + +@provider( + slots=[ + SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), + SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), + StringSlot(SPARSE_ID_LIMIT) + ], + use_seq=True, + init_hook=processSeqAndGenerateDataInit) def processSeqAndGenerateData(obj, name): retv = [sparse_value, dense, sparse_nonvalue, ids, strs] # Write to protoseq. @@ -104,10 +127,15 @@ def processSeqAndGenerateData(obj, name): def processSubSeqAndGenerateDataInit(obj, *args, **kwargs): obj.json_filename = kwargs.get("load_data_args", "test_data.json") -@provider(slots=[SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), - SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), - StringSlot(SPARSE_ID_LIMIT)], - use_seq=True, init_hook=processSubSeqAndGenerateDataInit) + +@provider( + slots=[ + SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), + SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), + StringSlot(SPARSE_ID_LIMIT) + ], + use_seq=True, + init_hook=processSubSeqAndGenerateDataInit) def processSubSeqAndGenerateData(obj, name): retv_json = [sparse_value, dense, sparse_nonvalue, ids, strs] retv_wrapper = [[sparse_value], [dense], [sparse_nonvalue], [ids], [strs]] @@ -116,6 +144,7 @@ def processSubSeqAndGenerateData(obj, name): json.dump(retv_json, f) yield retv_wrapper + if __name__ == "__main__": pvd = processNonSequenceData("test.txt") print pvd.getNextBatch(100) diff --git a/paddle/trainer/tests/test_gen_dict.txt b/paddle/trainer/tests/test_gen_dict.txt index 91a84146180e0135b37ee8c76508e588412c2870..1000f90057824bf665b32fe47a7f78e7a0077e7b 100644 --- a/paddle/trainer/tests/test_gen_dict.txt +++ b/paddle/trainer/tests/test_gen_dict.txt @@ -6,4 +6,4 @@ 5 6 7 -8 \ No newline at end of file +8 diff --git a/paddle/trainer/tests/train.txt b/paddle/trainer/tests/train.txt index 8d9b15dcf5bba477c2c6f732806a47b0aa6e098a..2313aee987ba71ba7ea779d3cf7705478e7fbde2 100644 --- a/paddle/trainer/tests/train.txt +++ b/paddle/trainer/tests/train.txt @@ -4998,4 +4998,3 @@ However RB B-ADVP the DT B-NP disclosure NN I-NP of IN B-PP - diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index 2cdff9d1aca927122fcdb0c2a7ab22a0e38b41c1..b16d4314654ffeab74137ec1ee69203dab56d851 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -378,7 +378,7 @@ hl_activation_mode_t hlActiveType(const std::string& type) { return HL_ACTIVATION_RELU; } else if (type == "tanh") { return HL_ACTIVATION_TANH; - } else if (type == "linear") { + } else if (type == "linear" || type == "") { return HL_ACTIVATION_LINEAR; } else { LOG(FATAL) << "Do not support activation type " << type; diff --git a/paddle/utils/enable_virtualenv.py b/paddle/utils/enable_virtualenv.py index 99d822a4145cca3f5ae35c4cf144210f35460827..ccfaa7c147b2ce25cb6007aa04cfc33961b7e10b 100644 --- a/paddle/utils/enable_virtualenv.py +++ b/paddle/utils/enable_virtualenv.py @@ -1,10 +1,12 @@ import os + def __activate_virtual_env__(): - __path__ = os.getenv('VIRTUAL_ENV') - if __path__ is None: - return - __script__ = os.path.join(__path__, 'bin', 'activate_this.py') - execfile(__script__, {'__file__': __script__}) + __path__ = os.getenv('VIRTUAL_ENV') + if __path__ is None: + return + __script__ = os.path.join(__path__, 'bin', 'activate_this.py') + execfile(__script__, {'__file__': __script__}) + __activate_virtual_env__() diff --git a/paddle/utils/tests/test_CommandLineParser.cpp b/paddle/utils/tests/test_CommandLineParser.cpp index d5f6018864cb9f14dd3006facebf82c66e909736..9bb6827540f61e8c6cc8b64c2b04ed4d0fcebab1 100644 --- a/paddle/utils/tests/test_CommandLineParser.cpp +++ b/paddle/utils/tests/test_CommandLineParser.cpp @@ -109,4 +109,3 @@ int main(int argc, char** argv) { } #endif - diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 479b457e55a7dac58ff390cce8d67d46da3b474d..aea77248cbac0f3ee044b05894d37718e692a0fc 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -120,6 +120,14 @@ message PoolConfig { optional uint32 padding_y = 13 [default = 0]; } +message SppConfig { + required string pool_type = 1; + required uint32 pyramid_height = 2; + required uint32 channels = 3; + required uint32 img_size = 4; + optional uint32 img_size_y = 5; +} + message NormConfig { // rnorm or cmrnorm required string norm_type = 1; @@ -196,6 +204,9 @@ message ProjectionConfig { // For IdentityOffsetProjection optional uint64 offset = 11 [default = 0]; + + // For pool + optional PoolConfig pool_conf = 12; } message OperatorConfig { @@ -245,6 +256,7 @@ message LayerInputConfig { optional string input_layer_argument = 9; optional BilinearInterpConfig bilinear_interp_conf = 10; optional MaxOutConfig maxout_conf = 11; + optional SppConfig spp_conf = 12; } message LayerConfig { diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 7f9e87eee6037666b86420fba194624859d356b3..c90af2ee000d46a032984ee23559e7e99b49ddad 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index 53409b746d811a3d73188a613c6b121e71955552..0c577ec657bc6d35c41e55ed5ab6adb80ab2c37c 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -18,9 +18,8 @@ import collections import functools import itertools -logging.basicConfig( - format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" - " %(message)s") +logging.basicConfig(format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" + " %(message)s") class SequenceType(object): @@ -132,8 +131,10 @@ class InputOrderWrapper(object): def __call__(self, obj, filename): for item in self.generator(obj, filename): if isinstance(item, dict): - yield [item.get(input_name, None) for input_name in - self.input_order] + yield [ + item.get(input_name, None) + for input_name in self.input_order + ] else: yield item @@ -162,8 +163,8 @@ class CheckWrapper(object): yield items except AssertionError as e: self.logger.warning( - "Item (%s) is not fit the input type with error %s" - % (repr(item), repr(e))) + "Item (%s) is not fit the input type with error %s" % + (repr(item), repr(e))) if self.check_fail_continue: continue @@ -202,13 +203,17 @@ class CheckWrapper(object): callback(each) -def provider(input_types=None, should_shuffle=None, pool_size=-1, +def provider(input_types=None, + should_shuffle=None, + pool_size=-1, min_pool_size=-1, can_over_batch_size=True, calc_batch_size=None, cache=CacheType.NO_CACHE, - check=False, check_fail_continue=False, - init_hook=None, **kwargs): + check=False, + check_fail_continue=False, + init_hook=None, + **kwargs): """ Provider decorator. Use it to make a function into PyDataProvider2 object. In this function, user only need to get each sample for some train/test @@ -318,9 +323,9 @@ def provider(input_types=None, should_shuffle=None, pool_size=-1, "Could not recognize should_shuffle (%s), " "just use default value of should_shuffle." " Please set should_shuffle to bool value or " - "something in %s" % ( - repr(self.should_shuffle), - repr(true_table + false_table))) + "something in %s" % + (repr(self.should_shuffle), + repr(true_table + false_table))) self.should_shuffle = None self.pool_size = pool_size @@ -351,8 +356,7 @@ def provider(input_types=None, should_shuffle=None, pool_size=-1, self.generator = InputOrderWrapper(self.generator, self.input_order) if self.check: - self.generator = CheckWrapper(self.generator, - self.slots, + self.generator = CheckWrapper(self.generator, self.slots, check_fail_continue, self.logger) @@ -368,4 +372,3 @@ def deserialize_args(args): :return: """ return cPickle.loads(args) - diff --git a/python/paddle/trainer/PyDataProviderWrapper.py b/python/paddle/trainer/PyDataProviderWrapper.py index c4b907af54699f31b3792fce423bc7251634e0da..90b684a000017fc03c8c33f829aaa64a5f769e45 100644 --- a/python/paddle/trainer/PyDataProviderWrapper.py +++ b/python/paddle/trainer/PyDataProviderWrapper.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ This module provide a wrapper(decorator) to wrap a data process method into a PyDataProvider. Some examples are shown `here `_. @@ -47,6 +46,7 @@ except ImportError: import io + class SlotType(object): # Just a hint for user. pass @@ -83,6 +83,7 @@ class SparseNonValueSlot(SlotType): - **SubSeq**: [[[int, int, ...], [int, ....], ...] , \ [[int, int, ...], [int, ....], ...] , ...] """ + def __init__(self, dim): """ :param dim: slot dimension @@ -294,8 +295,9 @@ class GeneralPyDataProvider: fn = "%s_%d" % (self.profile_filename, self.profile_count) sortby = "cumulative" with open(fn, "w") as f: - pstats.Stats(self.profiler, stream=f).sort_stats( - sortby).print_stats() + pstats.Stats( + self.profiler, + stream=f).sort_stats(sortby).print_stats() self.logger.info("saving profile to file %s" % fn) self.profile_count += 1 self.logger.info("resetting profile") @@ -453,9 +455,10 @@ class GeneralPyDataProvider: seq_stream.flush() subseq_stream.flush() - return "".join([self.int_packer.pack(current_batch_size), - data_bytes.getvalue(), - seq_bytes.getvalue(), subseq_bytes.getvalue()]) + return "".join([ + self.int_packer.pack(current_batch_size), data_bytes.getvalue(), + seq_bytes.getvalue(), subseq_bytes.getvalue() + ]) finally: data_stream.close() @@ -516,7 +519,7 @@ class GeneralPyDataProvider: self.data_pool[idx]) idx -= 1 - ret_list += self.data_pool[self.data_pool_idx: idx + 1] + ret_list += self.data_pool[self.data_pool_idx:idx + 1] # for speed reason, just shift left index, not delete data actually. self.data_pool_idx = idx + 1 @@ -537,8 +540,8 @@ class GeneralPyDataProvider: if self.max_pool_size == 0: for i in xrange(min(self.file_count, len(self.generators))): self.data_pool += list(self.generators[i]) - self.generators = self.generators[ - min(self.file_count, len(self.generators)):] + self.generators = self.generators[min(self.file_count, + len(self.generators)):] self.max_pool_size = len(self.data_pool) else: while len(self.data_pool) < self.max_pool_size and len( @@ -562,9 +565,15 @@ def default_init_hook(cls, *args, **kwargs): del cls, args, kwargs -def provider(slots=None, use_seq=False, should_shuffle=True, pool_size=1, - can_over_batch_size=True, calc_batch_size=lambda data: 1, - debug=False, init_hook=default_init_hook, profile_filename=None): +def provider(slots=None, + use_seq=False, + should_shuffle=True, + pool_size=1, + can_over_batch_size=True, + calc_batch_size=lambda data: 1, + debug=False, + init_hook=default_init_hook, + profile_filename=None): """ The decorator for PyDataProvider. User should use this to create Provider class. User should only concern how to read sample from file. @@ -663,7 +672,7 @@ def provider(slots=None, use_seq=False, should_shuffle=True, pool_size=1, def __init__(self, *file_list, **kwargs): logging.basicConfig( format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" - " %(message)s") + " %(message)s") self.logger = logging.getLogger("") if debug: diff --git a/python/paddle/trainer/__init__.py b/python/paddle/trainer/__init__.py index 7f9e87eee6037666b86420fba194624859d356b3..c90af2ee000d46a032984ee23559e7e99b49ddad 100644 --- a/python/paddle/trainer/__init__.py +++ b/python/paddle/trainer/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 4ac2b33f1ec06a5867b9b1051cc091f43a3e5811..dbe2f3b29278c259945564959690a3aa6c0cfbe0 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -13,7 +13,6 @@ # limitations under the License. from __future__ import print_function - ''' The following functions are available in the config file: @@ -101,50 +100,45 @@ except Exception as e: raise logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) __real_print__ = print -print=logger.info +print = logger.info # from layer type name to layer class g_layer_type_map = {} + # Initialize global variables. We use this function so that we can # call parse_config() multiple times def init_config_environment( - g_default_momentum = None, - g_default_decay_rate = None, - g_default_initial_mean = 0., - g_default_initial_std = 0.01, - g_default_num_batches_regularization = None, - g_default_initial_strategy = 0, - g_default_initial_smart = False, - g_default_gradient_clipping_threshold = None, - g_default_device = None, - g_default_update_hooks = None, - g_default_compact_func = None, - - g_config = TrainerConfig(), - g_layer_map = {}, - g_parameter_map = {}, - - g_extended_config_funcs = {}, + g_default_momentum=None, + g_default_decay_rate=None, + g_default_initial_mean=0., + g_default_initial_std=0.01, + g_default_num_batches_regularization=None, + g_default_initial_strategy=0, + g_default_initial_smart=False, + g_default_gradient_clipping_threshold=None, + g_default_device=None, + g_default_update_hooks=None, + g_default_compact_func=None, + g_config=TrainerConfig(), + g_layer_map={}, + g_parameter_map={}, + g_extended_config_funcs={}, # store command args of paddle_trainer - g_command_config_args = {}, + g_command_config_args={}, # Used for PyDataProvider to avoid duplicate module name - g_py_module_name_list = [], - - g_current_submodel = None, - g_root_submodel = None, - g_submodel_map = {}, - g_submodel_stack = [], - - g_add_submodel_suffix = False, - ): + g_py_module_name_list=[], + g_current_submodel=None, + g_root_submodel=None, + g_submodel_map={}, + g_submodel_stack=[], + g_add_submodel_suffix=False, ): for k, v in locals().iteritems(): globals()[k] = copy.deepcopy(v) @@ -161,43 +155,54 @@ def config_assert(b, msg): if not b: logger.fatal(msg) + g_config_funcs = {} + # decorator for indicating a function which can be used in config file def config_func(func): g_config_funcs[func.func_name] = func return func + # decorator for indicating a class which can be used in config file def config_class(cls): g_config_funcs[cls.__name__] = cls return cls + # decorator for indicating a class for a layer type def config_layer(layer_type): def wrap(cls): g_config_funcs[cls.__name__] = cls g_layer_type_map[layer_type] = cls return cls + return wrap + def gen_parameter_name(layer_name, input_index): return '_%s.w%d' % (layer_name, input_index) + def gen_bias_parameter_name(layer_name): return '_%s.wbias' % layer_name + def default(x, default_value): return default_value if x is None else x + class Cfg(object): def add_keys(self, locals): for k, v in locals.iteritems(): if not k.startswith('_'): self.__setattr__(k, v) + # functions available in config file + # Define the name of the input layers of the NeuralNetwork. # The type of these layers must be "data". # These layers will be provided with the DataBatch obtained @@ -216,9 +221,10 @@ def Inputs(*args): if g_current_submodel is g_root_submodel: g_config.model_config.input_layer_names.append(name) + @config_func def HasInputsSet(): - return len(g_config.model_config.input_layer_names) != 0 + return len(g_current_submodel.input_layer_names) != 0 # Define the name of the output layers of the NeuralNetwork. @@ -244,7 +250,7 @@ def SubModelBegin(name): global g_current_submodel, g_root_submodel, g_submodel_stack g_submodel_stack.append(g_current_submodel) - name = MakeLayerNameInParentSubmodel(name) #rename in nested submodel + name = MakeLayerNameInParentSubmodel(name) #rename in nested submodel config_assert(name not in g_submodel_map, 'Duplicated submodel name: %s' % name) @@ -254,36 +260,42 @@ def SubModelBegin(name): g_submodel_map[name] = sub_model g_current_submodel = sub_model + @config_func -def SubModelEnd(name = None): +def SubModelEnd(name=None): global g_current_submodel, g_root_submodel, g_submodel_stack - config_assert(g_current_submodel is not g_root_submodel, "submodel not begin") + config_assert(g_current_submodel is not g_root_submodel, + "submodel not begin") if name is not None: - config_assert(g_current_submodel.name == MakeLayerNameInParentSubmodel(name), - "submodel name error") + config_assert( + g_current_submodel.name == MakeLayerNameInParentSubmodel(name), + "submodel name error") g_current_submodel = g_submodel_stack.pop() + def MakeLayerNameInParentSubmodel(name): suffix = "" if len(g_submodel_stack) > 1: suffix = "@" + g_submodel_stack[-1].name return name + suffix + def GetLayerBaseName(name): return name.split('@')[0] -def MakeLayerNameInSubmodel(name, submodel_name = None): + +def MakeLayerNameInSubmodel(name, submodel_name=None): global g_current_submodel global g_add_submodel_suffix - if (submodel_name is None - and not g_add_submodel_suffix - and not g_current_submodel.is_recurrent_layer_group): + if (submodel_name is None and not g_add_submodel_suffix and + not g_current_submodel.is_recurrent_layer_group): return name if submodel_name is None: submodel_name = g_current_submodel.name return name + "@" + submodel_name + # Define a recurrent layer group begin with RecurrentLayerGroupBegin # and end with RecurrentLayerGroupEnd. # A recurrent layer group forward/backward one frame after previous frame @@ -332,8 +344,10 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, if in_links_count == 0: in_links_has_subseq = has_subseq else: - config_assert(in_links_has_subseq == has_subseq, - "The sequence type of in_links should be the same in RecurrentLayerGroup") + config_assert( + in_links_has_subseq == has_subseq, + "The sequence type of in_links should be the same in RecurrentLayerGroup" + ) in_links_count += 1 layer_name = MakeLayerNameInParentSubmodel(name) layer = g_layer_map[layer_name] @@ -347,6 +361,7 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, pair.link_name = MakeLayerNameInSubmodel(name) pair.has_subseq = has_subseq + @config_func def RecurrentLayerGroupSetOutLink(link): if isinstance(link, basestring): @@ -363,8 +378,7 @@ def RecurrentLayerGroupSetOutLink(link): def RecurrentLayerGroupSetGenerator(generator=None): - generator.eos_layer_name = MakeLayerNameInSubmodel( - generator.eos_layer_name) + generator.eos_layer_name = MakeLayerNameInSubmodel(generator.eos_layer_name) g_current_submodel.generator.CopyFrom(generator) @@ -375,21 +389,18 @@ def RecurrentLayerGroupBegin(name, generator=None, target_inlinkname="", seq_reversed=False): - RecurrentLayerGroupWithoutOutLinksBegin(name, - in_links, - seq_reversed, + RecurrentLayerGroupWithoutOutLinksBegin(name, in_links, seq_reversed, target_inlinkname) for link in out_links: RecurrentLayerGroupSetOutLink(link) - if generator is not None: RecurrentLayerGroupSetGenerator(generator) - config_assert(len(in_links) == 0, - "no in_links should be passed to generator") - config_assert(len(out_links) >= 1, - "one or more than one out_links should be passed to generator") - + config_assert( + len(in_links) == 0, "no in_links should be passed to generator") + config_assert( + len(out_links) >= 1, + "one or more than one out_links should be passed to generator") @config_func @@ -397,9 +408,10 @@ def RecurrentLayerGroupEnd(name): global g_current_submodel config_assert(g_current_submodel.is_recurrent_layer_group, "RecurrentLayerGroup not begin") - for pair in g_current_submodel.memories: #check exist + for pair in g_current_submodel.memories: #check exist layer = g_layer_map[pair.layer_name] - config_assert(layer is not None, "memory declare wrong name:%s" % pair.layer_name) + config_assert(layer is not None, + "memory declare wrong name:%s" % pair.layer_name) memory_link = g_layer_map[pair.link_name] config_assert(layer.size == memory_link.size, "memory declare wrong size:%d" % memory_link.size) @@ -418,12 +430,14 @@ def RecurrentLayerGroupEnd(name): else: GatherAgentLayer(name=agent_name, size=layer.size) + # Define the model type # currently, the paddle supports "nn", "recurrent_nn", "recursive_nn" and "multi_nn" @config_func def model_type(name): g_config.model_config.type = name + @config_class class Bias(Cfg): def __init__( @@ -441,10 +455,10 @@ class Bias(Cfg): sparse_remote_update=None, gradient_clipping_threshold=None, is_static=None, - is_shared=None, - ): + is_shared=None, ): self.add_keys(locals()) + # Define one input for a layer @config_class class Input(Cfg): @@ -471,24 +485,26 @@ class Input(Cfg): image=None, block_expand=None, maxout=None, + spp=None, format=None, nnz=None, is_static=None, is_shared=None, update_hooks=None, - input_layer_argument=None, - ): + input_layer_argument=None, ): self.add_keys(locals()) self.input_layer_name = MakeLayerNameInSubmodel(input_layer_name) + # Define a projection for iexed layer @config_class class Projection(Input): - type = None # subclass should set it correctly + type = None # subclass should set it correctly + def __init__( self, input_layer_name, - size = 0, # projection output size + size=0, # projection output size parameter_name=None, learning_rate=None, momentum=None, @@ -508,8 +524,7 @@ class Projection(Input): is_static=None, is_shared=None, update_hooks=None, - input_layer_argument=None, - ): + input_layer_argument=None, ): self.add_keys(locals()) self.input_layer_name = MakeLayerNameInSubmodel(input_layer_name) @@ -523,8 +538,10 @@ class Projection(Input): # to indicate using the size from Layer config def calc_output_size(self, input_layer_config): return self.size + def calc_parameter_size(self, input_size, output_size): raise NotimplementedError + def calc_parameter_dims(self, input_size, output_size): raise NotimplementedError @@ -535,31 +552,32 @@ class IdentityProjection(Projection): def calc_output_size(self, input_layer_config): return input_layer_config.size + def calc_parameter_size(self, input_size, output_size): return 0 + def calc_parameter_dims(self, input_size, output_size): return [] + # Like IdentityProjection, but layer size may smaller than input size, # the projection select dimesions [offset, offset+layer_size) from input @config_class class IdentityOffsetProjection(Projection): type = 'identity_offset' - def __init__( - self, - input_layer_name, - offset, - **xargs): - super(IdentityOffsetProjection, self).__init__( - input_layer_name, **xargs) + def __init__(self, input_layer_name, offset, **xargs): + super(IdentityOffsetProjection, self).__init__(input_layer_name, + **xargs) self.proj_conf.offset = offset def calc_parameter_size(self, input_size, output_size): return 0 + def calc_parameter_dims(self, input_size, output_size): return [] + # DotMulProjection performs element-wise multiplication with weight @config_class class DotMulProjection(Projection): @@ -567,49 +585,67 @@ class DotMulProjection(Projection): def calc_output_size(self, input_layer_config): return input_layer_config.size + def calc_parameter_size(self, input_size, output_size): return output_size + def calc_parameter_dims(self, input_size, output_size): return [1, output_size] +# ScalingProjection +@config_class +class ScalingProjection(Projection): + type = 'scaling' + + def calc_output_size(self, input_layer_config): + return input_layer_config.size + + def calc_parameter_size(self, input_size, output_size): + return 1 + + def calc_parameter_dims(self, input_size, output_size): + return [1, 1] + + @config_class class TableProjection(Projection): type = 'table' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [input_size, output_size] + @config_class class FullMatrixProjection(Projection): type = 'fc' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [input_size, output_size] + @config_class class TransposedFullMatrixProjection(Projection): type = 'trans_fc' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [output_size, input_size] + @config_class class ContextProjection(Projection): type = 'context' - def __init__( - self, - input_layer_name, - context_start, - context_length, - trainable_padding, - **xargs): + def __init__(self, input_layer_name, context_start, context_length, + trainable_padding, **xargs): super(ContextProjection, self).__init__(input_layer_name, **xargs) self.proj_conf.context_start = context_start self.proj_conf.context_length = context_length @@ -637,22 +673,21 @@ class ContextProjection(Projection): class ConvProjection(Projection): type = 'conv' - def __init__( - self, - input_layer_name, - num_filters=None, - conv_conf=None, - **xargs): + def __init__(self, + input_layer_name, + num_filters=None, + conv_conf=None, + **xargs): super(ConvProjection, self).__init__(input_layer_name, **xargs) if num_filters is not None: self.proj_conf.num_filters = num_filters - parse_conv(conv_conf, - input_layer_name, - self.proj_conf.conv_conf) + parse_conv(conv_conf, input_layer_name, self.proj_conf.conv_conf, + num_filters) # TODO: support rectangle input - self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x ** 2) * num_filters + self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x + **2) * num_filters def calc_output_size(self, input_layer_config): return self.proj_conf.output_size @@ -674,11 +709,11 @@ class ConvProjection(Projection): # Define a operator for mixed layer @config_class class Operator(Cfg): - type = None # subclass should set it correctly + type = None # subclass should set it correctly + def __init__( self, - input_layer_names, - ): + input_layer_names, ): self.add_keys(locals()) self.operator_conf = OperatorConfig() self.operator_conf.type = self.type @@ -689,16 +724,13 @@ class Operator(Cfg): def calc_output_size(self, input_sizes): return 0 + @config_class class DotMulOperator(Operator): type = 'dot_mul' - def __init__( - self, - input_layer_names, - scale=None, - **xargs): - super(DotMulOperator, self).__init__( - input_layer_names, **xargs) + + def __init__(self, input_layer_names, scale=None, **xargs): + super(DotMulOperator, self).__init__(input_layer_names, **xargs) if scale is not None: self.operator_conf.dotmul_scale = scale @@ -714,25 +746,24 @@ class DotMulOperator(Operator): return input_sizes[0] - @config_class class ConvOperator(Operator): type = 'conv' - def __init__( - self, - input_layer_names, - num_filters=None, - conv_conf=None, - **xargs): - super(ConvOperator, self).__init__( - input_layer_names, **xargs) + + def __init__(self, + input_layer_names, + num_filters=None, + conv_conf=None, + **xargs): + super(ConvOperator, self).__init__(input_layer_names, **xargs) if num_filters is not None: self.operator_conf.num_filters = num_filters parse_conv(conv_conf, MakeLayerNameInSubmodel(input_layer_names[0]), - self.operator_conf.conv_conf) - self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x ** 2) * num_filters + self.operator_conf.conv_conf, num_filters) + self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x + **2) * num_filters config_assert(len(input_layer_names) == 2, "Conv is binary operator") @@ -743,108 +774,106 @@ class ConvOperator(Operator): # please refer to the comments in proto/ModelConfig.proto @config_class class Conv(Cfg): - def __init__( - self, - filter_size, - channels, - padding = None, - stride = None, - groups = None, - filter_channels = None, - output_x = None, - img_size = None, - caffe_mode = True, - filter_size_y = None, - padding_y = None, - stride_y = None): + def __init__(self, + filter_size, + channels, + padding=None, + stride=None, + groups=None, + filter_channels=None, + output_x=None, + img_size=None, + caffe_mode=True, + filter_size_y=None, + padding_y=None, + stride_y=None): self.add_keys(locals()) if filter_size_y is None: - self.filter_size_y = filter_size + self.filter_size_y = filter_size if padding_y is None: - self.padding_y = padding + self.padding_y = padding if stride_y is None: - self.stride_y = stride + self.stride_y = stride if output_x is not None: - config_assert(output_x <= 0) + config_assert(output_x <= 0) + # please refer to the comments in proto/ModelConfig.proto @config_class class BilinearInterp(Cfg): - def __init__( - self, - out_size_x = None, - out_size_y = None, - num_channels = None): + def __init__(self, out_size_x=None, out_size_y=None, num_channels=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Pool(Cfg): - def __init__( - self, - pool_type, - channels, - size_x, - size_y = None, - img_width = None, - start = None, - stride = None, - stride_y = None, - padding = None, - padding_y = None): + def __init__(self, + pool_type, + channels, + size_x, + size_y=None, + img_width=None, + start=None, + stride=None, + stride_y=None, + padding=None, + padding_y=None): self.add_keys(locals()) + +# please refer to the comments in proto/ModelConfig.proto +@config_class +class SpatialPyramidPool(Cfg): + def __init__(self, pool_type, pyramid_height, channels, img_width=None): + self.add_keys(locals()) + + # please refer to the comments in proto/ModelConfig.proto @config_class class Norm(Cfg): - def __init__( - self, - norm_type, - channels, - size, - scale, - pow, - output_x = None, - img_size = None, - blocked = None): + def __init__(self, + norm_type, + channels, + size, + scale, + pow, + output_x=None, + img_size=None, + blocked=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Image(Cfg): - def __init__( - self, - channels, - img_size = None): + def __init__(self, channels, img_size=None): self.add_keys(locals()) + @config_class class BlockExpand(Cfg): - def __init__( - self, - channels, - padding_x = 0, - padding_y = 0, - stride_x = 0, - stride_y = 0, - block_x = 0, - block_y = 0, - img_size_x = 0, - img_size_y = 0, - output_x = 0, - output_y = 0): + def __init__(self, + channels, + padding_x=0, + padding_y=0, + stride_x=0, + stride_y=0, + block_x=0, + block_y=0, + img_size_x=0, + img_size_y=0, + output_x=0, + output_y=0): self.add_keys(locals()) + @config_class class MaxOut(Cfg): - def __init__( - self, - channels, - groups, - img_size_x = 0, - img_size_y = 0): + def __init__(self, channels, groups, img_size_x=0, img_size_y=0): self.add_keys(locals()) + def DataBase(async_load_data=False, constant_slots=None, data_ratio=1, @@ -858,23 +887,23 @@ def DataBase(async_load_data=False, if constant_slots: data_config.constant_slots.extend(constant_slots) - data_config.data_ratio=data_ratio - data_config.is_main_data=is_main_data + data_config.data_ratio = data_ratio + data_config.is_main_data = is_main_data - usage_ratio=default(usage_ratio, settings_deprecated["usage_ratio"]) + usage_ratio = default(usage_ratio, settings_deprecated["usage_ratio"]) config_assert(usage_ratio >= 0 and usage_ratio <= 1, "The range of usage_ratio is [0, 1]") data_config.usage_ratio = usage_ratio return data_config + @config_func -def SimpleData( - files=None, - feat_dim=None, - context_len=None, - buffer_capacity=None, - **xargs): +def SimpleData(files=None, + feat_dim=None, + context_len=None, + buffer_capacity=None, + **xargs): data_config = DataBase(**xargs) data_config.type = 'simple' data_config.files = files @@ -885,31 +914,36 @@ def SimpleData( data_config.buffer_capacity = buffer_capacity return data_config + @config_func -def PyData( - files=None, - type=None, - file_group_queue_capacity=None, - load_data_module=None, - load_data_object=None, - load_data_args="", - load_file_count=None, - constant_slots=None, - load_thread_num=None, - **xargs): +def PyData(files=None, + type=None, + file_group_queue_capacity=None, + load_data_module=None, + load_data_object=None, + load_data_args="", + load_file_count=None, + constant_slots=None, + load_thread_num=None, + **xargs): data_config = DataBase(**xargs) data_config.type = 'py' if load_data_module in g_py_module_name_list: + def get_path(module): m = __import__(load_data_module) return os.path.split(os.path.realpath(m.__file__))[0] + # python C-api is not thread safe, one module can only be import once, # so here we nedd to copy the module with different names if it has to be # imported several times. - module_new_name = "%s_copy_%d" % (load_data_module, len(g_py_module_name_list)) + module_new_name = "%s_copy_%d" % (load_data_module, + len(g_py_module_name_list)) g_py_module_name_list.append(module_new_name) - module_path = "%s/%s.py" % (get_path(load_data_module), load_data_module) - new_module_path = "%s/%s.py" % (get_path(load_data_module), module_new_name) + module_path = "%s/%s.py" % (get_path(load_data_module), + load_data_module) + new_module_path = "%s/%s.py" % (get_path(load_data_module), + module_new_name) if os.path.isfile(module_path) == False: raise Exception("File %s is not exist." % module_path) shutil.copy2(module_path, new_module_path) @@ -934,15 +968,15 @@ def PyData( data_config.constant_slots.extend(constant_slots) return data_config + @config_func -def ProtoData( - files=None, - type=None, - file_group_queue_capacity=None, - load_file_count=None, - constant_slots=None, - load_thread_num=None, - **xargs): +def ProtoData(files=None, + type=None, + file_group_queue_capacity=None, + load_file_count=None, + constant_slots=None, + load_thread_num=None, + **xargs): data_config = DataBase(**xargs) if type is None: data_config.type = 'proto' @@ -963,25 +997,24 @@ def ProtoData( data_config.constant_slots.extend(constant_slots) return data_config + #real data for training is actually provided by "sub_data" data providers. @config_func -def MultiData( - sub_data=[] - ): +def MultiData(sub_data=[]): data_config = DataConfig() data_config.type = 'multi' data_config.sub_data_configs.extend(sub_data) return data_config + @config_func -def Data( - type, - files=None, - feat_dim=None, - slot_dims=None, - context_len=None, - buffer_capacity=None, - **xargs): +def Data(type, + files=None, + feat_dim=None, + slot_dims=None, + context_len=None, + buffer_capacity=None, + **xargs): data_config = DataBase(**xargs) data_config.type = type @@ -1017,15 +1050,19 @@ def TestData(data_config, async_load_data=None): " Data definition") g_config.test_data_config.async_load_data = async_load_data + def parse_bilinear(bilinear, input_layer_name, bilinear_conf): - bilinear_conf.out_size_x = bilinear.out_size_x; - bilinear_conf.out_size_y = bilinear.out_size_y; - bilinear_conf.num_channels = bilinear.num_channels; + bilinear_conf.out_size_x = bilinear.out_size_x + bilinear_conf.out_size_y = bilinear.out_size_y + bilinear_conf.num_channels = bilinear.num_channels + ''' caffe_mode: compute the output size using floor instead of ceil, which is consistent of caffe and CuDNN's convention. ''' + + def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): output = (2 * padding + img_size - filter_size) / float(stride) if caffe_mode: @@ -1033,54 +1070,89 @@ def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): else: return 1 + int(math.ceil(output)) + +''' +calcualte image_size based on output_size for convolution. +It is the reverse function of cnn_output_size +''' + + +def cnn_image_size(output_size, filter_size, padding, stride, caffe_mode): + if caffe_mode: + img_size = (output_size - 1) * stride + filter_size - 2 * padding + else: + img_size = (output_size - 2) * stride + filter_size - 2 * padding + 1 + return img_size + + def parse_pool(pool, input_layer_name, pool_conf): pool_conf.pool_type = pool.pool_type - config_assert(pool.pool_type in ['max-projection', 'avg-projection', - 'cudnn-max-pool', 'cudnn-avg-pool'], - "pool-type %s is not in " + config_assert(pool.pool_type in [ + 'max-projection', 'avg-projection', 'cudnn-max-pool', 'cudnn-avg-pool' + ], "pool-type %s is not in " "['max-projection', 'avg-projection', " - "'cudnn-max-pool', 'cudnn-avg-pool']" - % pool.pool_type) + "'cudnn-max-pool', 'cudnn-avg-pool']" % pool.pool_type) pool_conf.channels = pool.channels pool_conf.size_x = pool.size_x pool_conf.stride = pool.stride pool_conf.size_y = default(pool.size_y, pool_conf.size_x) - pool_conf.stride_y = default(pool.stride_y, pool_conf.stride); + pool_conf.stride_y = default(pool.stride_y, pool_conf.stride) img_pixels = g_layer_map[input_layer_name].size / pool.channels # the img_width may be removed, # and it can be calculated automatically later. - pool_conf.img_size = default(pool.img_width, int(img_pixels ** 0.5)) + pool_conf.img_size = default(pool.img_width, int(img_pixels**0.5)) pool_conf.img_size_y = img_pixels / pool_conf.img_size config_assert(pool_conf.img_size * pool_conf.img_size_y == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (pool_conf.img_size, img_pixels)) + "Incorrect input image size %d for input image pixels %d" % + (pool_conf.img_size, img_pixels)) config_assert(not pool.start, "start is deprecated in pooling.") if pool.padding is not None: pool_conf.padding = pool.padding pool_conf.padding_y = default(pool.padding_y, pool_conf.padding) - pool_conf.output_x = cnn_output_size(pool_conf.img_size, pool_conf.size_x, - pool_conf.padding, pool_conf.stride, False) - pool_conf.output_y = cnn_output_size(pool_conf.img_size_y, pool_conf.size_y, - pool_conf.padding_y, pool_conf.stride_y, False) + pool_conf.output_x = cnn_output_size( + pool_conf.img_size, pool_conf.size_x, pool_conf.padding, + pool_conf.stride, False) + pool_conf.output_y = cnn_output_size( + pool_conf.img_size_y, pool_conf.size_y, pool_conf.padding_y, + pool_conf.stride_y, False) + + +def parse_spp(spp, input_layer_name, spp_conf): + spp_conf.pool_type = spp.pool_type + config_assert(spp.pool_type in ['max-projection', 'avg-projection'], + "pool-type %s is not in " + "['max-projection', 'avg-projection']" % spp.pool_type) + spp_conf.pyramid_height = spp.pyramid_height + spp_conf.channels = spp.channels + + img_pixels = g_layer_map[input_layer_name].size / spp_conf.channels + + spp_conf.img_size = default(spp.img_width, int(img_pixels**0.5)) + spp_conf.img_size_y = img_pixels / spp_conf.img_size + config_assert(spp_conf.img_size * spp_conf.img_size_y == img_pixels, + "Incorrect input image size %d for input image pixels %d" % + (spp_conf.img_size, img_pixels)) + def parse_image(image, input_layer_name, image_conf): image_conf.channels = image.channels image_pixels = g_layer_map[input_layer_name].size / image_conf.channels - image_conf.img_size = int(image_pixels ** 0.5) - config_assert((image_conf.img_size ** 2) == image_pixels, - "Incorrect input image size %d for input image pixels %d" - % (image_conf.img_size, image_pixels)) + image_conf.img_size = int(image_pixels**0.5) + config_assert((image_conf.img_size**2) == image_pixels, + "Incorrect input image size %d for input image pixels %d" % + (image_conf.img_size, image_pixels)) + def parse_norm(norm, input_layer_name, norm_conf): norm_conf.norm_type = norm.norm_type config_assert(norm.norm_type in ['rnorm', 'cmrnorm-projection'], - "norm-type %s is not in [rnorm, 'cmrnorm-projection']" - % norm.norm_type) + "norm-type %s is not in [rnorm, 'cmrnorm-projection']" % + norm.norm_type) norm_conf.channels = norm.channels norm_conf.size = norm.size norm_conf.scale = norm.scale @@ -1088,17 +1160,24 @@ def parse_norm(norm, input_layer_name, norm_conf): norm_conf.blocked = norm.blocked img_pixels = g_layer_map[input_layer_name].size / norm.channels - norm_conf.img_size = int(img_pixels ** 0.5) - config_assert((norm_conf.img_size ** 2) == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (norm_conf.img_size, img_pixels)) + norm_conf.img_size = int(img_pixels**0.5) + config_assert((norm_conf.img_size**2) == img_pixels, + "Incorrect input image size %d for input image pixels %d" % + (norm_conf.img_size, img_pixels)) norm_conf.output_x = norm_conf.img_size if norm.norm_type in ['cmrnorm-projection']: norm_conf.scale /= norm.size else: - norm_conf.scale /= norm.size ** 2 + norm_conf.scale /= norm.size**2 -def parse_conv(conv, input_layer_name, conv_conf): + +''' +caffe_mode: compute the output size using floor instead of ceil, + which is consistent of caffe and CuDNN's convention. +''' + + +def parse_conv(conv, input_layer_name, conv_conf, num_filters, trans=False): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y conv_conf.channels = conv.channels @@ -1107,20 +1186,38 @@ def parse_conv(conv, input_layer_name, conv_conf): conv_conf.stride = conv.stride conv_conf.stride_y = conv.stride_y conv_conf.groups = conv.groups - conv_conf.filter_channels = conv.channels / conv.groups conv_conf.caffe_mode = conv.caffe_mode - img_pixels = g_layer_map[input_layer_name].size / conv.channels - print('channels=%d size=%d'%(conv.channels, - g_layer_map[input_layer_name].size)) - conv_conf.img_size = int(img_pixels ** 0.5) - config_assert((conv_conf.img_size ** 2) == img_pixels, - ("Input layer %s: Incorrect input image size %d for input " - + "image pixels %d") - % (input_layer_name, conv_conf.img_size, img_pixels)) - conv_conf.output_x = cnn_output_size(conv_conf.img_size, conv_conf.filter_size, - conv_conf.padding, conv_conf.stride, - conv_conf.caffe_mode) + if not trans: + conv_conf.filter_channels = conv.channels / conv.groups + + img_pixels = g_layer_map[input_layer_name].size / conv.channels + print('channels=%d size=%d' % (conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.img_size = int(img_pixels**0.5) + config_assert((conv_conf.img_size**2) == img_pixels, ( + "Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") % + (input_layer_name, conv_conf.img_size, img_pixels)) + + conv_conf.output_x = cnn_output_size( + conv_conf.img_size, conv_conf.filter_size, conv_conf.padding, + conv_conf.stride, conv_conf.caffe_mode) + else: + conv_conf.filter_channels = num_filters / conv.groups + + outputSize = g_layer_map[input_layer_name].size / conv.channels + print('channels=%d size=%d' % (conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.output_x = int(outputSize**0.5) + config_assert((conv_conf.output_x**2) == outputSize, ( + "Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") % + (input_layer_name, conv_conf.output_x, outputSize)) + conv_conf.img_size = cnn_image_size( + conv_conf.output_x, conv_conf.filter_size, conv_conf.padding, + conv_conf.stride, conv_conf.caffe_mode) + def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.channels = block_expand.channels @@ -1136,37 +1233,38 @@ def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.output_x = 0 else: block_expand_conf.output_x = cnn_output_size( - block_expand.img_size_x, block_expand.block_x, + block_expand.img_size_x, block_expand.block_x, block_expand.padding_x, block_expand.stride_x, False) if block_expand_conf.img_size_y == 0: block_expand_conf.output_y = 0 else: block_expand_conf.output_y = cnn_output_size( - block_expand.img_size_y, block_expand.block_y, + block_expand.img_size_y, block_expand.block_y, block_expand.padding_y, block_expand.stride_y, False) + def parse_maxout(maxout, input_layer_name, maxout_conf): maxout_conf.channels = maxout.channels maxout_conf.groups = maxout.groups maxout_conf.img_size_x = maxout.img_size_x maxout_conf.img_size_y = maxout.img_size_y - + + # Define an evaluator @config_func def Evaluator( name, type, inputs, - chunk_scheme = None, - num_chunk_types = None, - classification_threshold = None, - positive_label = None, - dict_file = None, - result_file = None, - num_results = None, - delimited = None, - ): + chunk_scheme=None, + num_chunk_types=None, + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None, ): evaluator = g_config.model_config.evaluators.add() evaluator.type = type evaluator.name = MakeLayerNameInSubmodel(name) @@ -1195,19 +1293,20 @@ def Evaluator( if delimited is not None: evaluator.delimited = delimited + class LayerBase(object): def __init__( self, name, type, - size, # size can be 0. In this case, subclass should set it. + size, # size can be 0. In this case, subclass should set it. inputs, device=None, active_type="", drop_rate=0., coeff=None): config_assert('@' not in name, - "layer name: %s contain special character @" % name) + "layer name: %s contain special character @" % name) global g_current_submodel name = MakeLayerNameInSubmodel(name) @@ -1246,8 +1345,8 @@ class LayerBase(object): if type_of(input) == str: input_layer_name = input input_config = Input( - input_layer_name = input, - parameter_name = gen_parameter_name(name, input_index)) + input_layer_name=input, + parameter_name=gen_parameter_name(name, input_index)) input_layer_name = input_config.input_layer_name elif isinstance(input, Input): input_layer_name = input.input_layer_name @@ -1256,16 +1355,15 @@ class LayerBase(object): input_config.parameter_name = \ gen_parameter_name(name, input_index) elif isinstance(input, Operator): - self.operators.append(input); + self.operators.append(input) input.operator_conf.input_indices.append(input_index) input_config = Input(input.input_layer_names[0]) input_layer_name = input_config.input_layer_name else: - raise ValueError( - 'Wrong type for inputs: %s' % type_of(input)) + raise ValueError('Wrong type for inputs: %s' % type_of(input)) config_assert(input_layer_name in g_layer_map, - "Unknown input layer '%s' for layer %s" - % (input_layer_name, name)) + "Unknown input layer '%s' for layer %s" % + (input_layer_name, name)) self.inputs[input_index] = input_config layer_input = self.config.inputs.add() layer_input.input_layer_name = input_config.input_layer_name @@ -1277,26 +1375,26 @@ class LayerBase(object): g_current_submodel.layer_names.append(self.config.name) - def get_input_layer(self, input_index): return g_layer_map[self.config.inputs[input_index].input_layer_name] # will return the bias created if not *for_self* def create_bias_parameter( self, - bias, # True/False or BiasCfg + bias, # True/False or BiasCfg size, - dims = None, - for_self = True, # whether create bias for layer self - ): + dims=None, + for_self=True, # whether create bias for layer self + ): if size == 0: return if dims is None: dims = [1, size] - config_assert(type_of(bias) == bool or type_of(bias) == Bias, - 'Incorrect type for bias: %s' % type_of(bias)) + config_assert( + type_of(bias) == bool or type_of(bias) == Bias, + 'Incorrect type for bias: %s' % type_of(bias)) if type_of(bias) == bool: if bias: @@ -1311,7 +1409,8 @@ class LayerBase(object): Parameter( bias.parameter_name, size, - self.config.device if self.config.HasField('device') else None, + self.config.device + if self.config.HasField('device') else None, dims, bias.learning_rate, bias.momentum, @@ -1323,22 +1422,21 @@ class LayerBase(object): initial_smart=bias.initial_smart, num_batches_regularization=bias.num_batches_regularization, sparse_remote_update=bias.sparse_remote_update, - gradient_clipping_threshold=bias.gradient_clipping_threshold, + gradient_clipping_threshold=bias. + gradient_clipping_threshold, is_static=bias.is_static, - is_shared=bias.is_shared, - ) + is_shared=bias.is_shared, ) if for_self: self.config.bias_parameter_name = bias.parameter_name else: return bias.parameter_name - def create_input_parameter( - self, - input_index, - size, - dims=None, - sparse = None, - format = None): + def create_input_parameter(self, + input_index, + size, + dims=None, + sparse=None, + format=None): if dims is None: # TODO(yuyang18): print warning and callstack here! dims = list() @@ -1353,12 +1451,12 @@ class LayerBase(object): if input_config.parameter_name in g_parameter_map: para = g_parameter_map[input_config.parameter_name] - config_assert(size == para.size, ('Shared parameter "%s" does not ' - + 'have same size: %s vs. %s') + config_assert(size == para.size, ( + 'Shared parameter "%s" does not ' + 'have same size: %s vs. %s') % (input_config.parameter_name, para.size, size)) - config_assert(dims == para.dims, ('Shared parameter "%s" does not ' - + 'have same dims: %s vs. %s') + config_assert(dims == para.dims, ( + 'Shared parameter "%s" does not ' + 'have same dims: %s vs. %s') % (input_config.parameter_name, para.dims, dims)) return @@ -1378,13 +1476,13 @@ class LayerBase(object): num_batches_regularization=input_config.num_batches_regularization, sparse_remote_update=input_config.sparse_remote_update, sparse_update=input_config.sparse_update, - gradient_clipping_threshold=input_config.gradient_clipping_threshold, + gradient_clipping_threshold=input_config. + gradient_clipping_threshold, sparse=sparse, format=format, is_static=input_config.is_static, is_shared=input_config.is_shared, - update_hooks=input_config.update_hooks - ) + update_hooks=input_config.update_hooks) def set_layer_size(self, size): if self.config.size == 0: @@ -1394,27 +1492,18 @@ class LayerBase(object): 'Different inputs result in' + 'different layer size at layer %s' % self.config.name) + @config_layer('multi_class_cross_entropy_with_selfnorm') class MultiClassCrossEntropySelfNormCostLayer(LayerBase): - def __init__( - self, - name, - inputs, - softmax_selfnorm_alpha=0.1, - **xargs): - super(MultiClassCrossEntropySelfNormCostLayer, self).__init__(name, - 'multi_class_cross_entropy_with_selfnorm', 0, inputs, **xargs) + def __init__(self, name, inputs, softmax_selfnorm_alpha=0.1, **xargs): + super(MultiClassCrossEntropySelfNormCostLayer, self).__init__( + name, 'multi_class_cross_entropy_with_selfnorm', 0, inputs, **xargs) self.config.softmax_selfnorm_alpha = softmax_selfnorm_alpha + @config_layer('fc') class FCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - bias=True, - **xargs): + def __init__(self, name, size, inputs, bias=True, **xargs): super(FCLayer, self).__init__(name, 'fc', size, inputs=inputs, **xargs) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) @@ -1428,22 +1517,23 @@ class FCLayer(LayerBase): else: sparse = None - self.create_input_parameter(input_index, psize, dims, sparse, format) + self.create_input_parameter(input_index, psize, dims, sparse, + format) self.create_bias_parameter(bias, self.config.size) + @config_layer('selective_fc') class SelectiveFCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - bias=True, - selective_fc_pass_generation=False, - has_selected_colums=True, - selective_fc_full_mul_ratio=0.02, - selective_fc_parallel_plain_mul_thread_num=None, - **xargs): + def __init__(self, + name, + size, + inputs, + bias=True, + selective_fc_pass_generation=False, + has_selected_colums=True, + selective_fc_full_mul_ratio=0.02, + selective_fc_parallel_plain_mul_thread_num=None, + **xargs): super(SelectiveFCLayer, self).__init__( name, 'selective_fc', size, inputs=inputs, **xargs) # user MUST know if selctive fc is used in training, @@ -1464,8 +1554,8 @@ class SelectiveFCLayer(LayerBase): input_num = len(self.inputs) if has_selected_colums: config_assert(input_num >= 2, - ("if indices of selected columns are not specified, " - "selective_fc Layer has at least two inputs")) + ("if indices of selected columns are not specified, " + "selective_fc Layer has at least two inputs")) input_num -= 1 for input_index in xrange(input_num): @@ -1478,26 +1568,23 @@ class SelectiveFCLayer(LayerBase): if sparse: psize = self.inputs[input_index].nnz - self.create_input_parameter( - input_index, psize, dims, sparse, format) + self.create_input_parameter(input_index, psize, dims, sparse, + format) self.create_bias_parameter(bias, self.config.size) + @config_layer('print') class PrintLayer(LayerBase): - def __init__( - self, - name, - inputs): + def __init__(self, name, inputs): super(PrintLayer, self).__init__(name, 'print', 0, inputs) + @config_layer('data') class DataLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): - super(DataLayer, self).__init__(name, 'data' , size, inputs=[], device=device) + def __init__(self, name, size, device=None): + super(DataLayer, self).__init__( + name, 'data', size, inputs=[], device=device) + ''' DataNormLayer: A layer for data normalization @@ -1525,14 +1612,11 @@ Note: min-max: y = (x-min)/(max-min) decimal-scaling: y = x/10^j, where j is the smallest integer such that max(|y|)<1 ''' + + @config_layer('data_norm') class DataNormLayer(LayerBase): - def __init__( - self, - name, - inputs, - data_norm_strategy="z-score", - device=None): + def __init__(self, name, inputs, data_norm_strategy="z-score", device=None): super(DataNormLayer, self).__init__( name, 'data_norm', 0, inputs=inputs, device=device) self.config.data_norm_strategy = data_norm_strategy @@ -1544,15 +1628,12 @@ class DataNormLayer(LayerBase): self.inputs[0].is_static = True self.create_input_parameter(0, para_size, para_dims) + @config_layer('prelu') class ParameterReluLayer(LayerBase): layer_type = 'prelu' - def __init__( - self, - name, - inputs, - partial_sum = 1, - **args): + + def __init__(self, name, inputs, partial_sum=1, **args): super(ParameterReluLayer, self).__init__( name, self.layer_type, 0, inputs=inputs, **args) config_assert(len(self.inputs) == 1) @@ -1561,17 +1642,18 @@ class ParameterReluLayer(LayerBase): self.set_layer_size(input_layer.size) self.create_input_parameter(0, input_layer.size / partial_sum) + @config_layer('conv') class ConvLayerBase(LayerBase): layer_type = 'conv' - def __init__( - self, - name, - inputs=[], - bias=True, - num_filters=None, - shared_biases=False, - **xargs): + + def __init__(self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): super(ConvLayerBase, self).__init__( name, self.layer_type, 0, inputs=inputs, **xargs) @@ -1588,7 +1670,7 @@ class ConvLayerBase(LayerBase): config_assert(use_gpu, "cudnn_conv only support GPU") if (use_gpu == 1 and self.layer_type != "exconv" and - (parallel_nn == 0 or self.config.device > -1)): + (parallel_nn == 0 or self.config.device > -1)): self.layer_type = "cudnn_conv" else: self.layer_type = "exconv" @@ -1600,16 +1682,14 @@ class ConvLayerBase(LayerBase): for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_conv( - self.inputs[input_index].conv, - input_layer.name, - self.config.inputs[input_index].conv_conf) + parse_conv(self.inputs[input_index].conv, input_layer.name, + self.config.inputs[input_index].conv_conf, num_filters) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) print("output size for %s is %d " % (name, conv_conf.output_x)) self.create_input_parameter(input_index, psize) self.set_layer_size( - (conv_conf.output_x ** 2) * self.config.num_filters) + (conv_conf.output_x**2) * self.config.num_filters) psize = self.config.size if shared_biases: @@ -1620,70 +1700,139 @@ class ConvLayerBase(LayerBase): return self.config.num_filters * conv_conf.filter_channels \ * (conv_conf.filter_size * conv_conf.filter_size_y) + @config_layer('exconv') class ConvLayer(ConvLayerBase): layer_type = 'exconv' + @config_layer('cudnn_conv') class ConvLayer(ConvLayerBase): layer_type = 'cudnn_conv' + +@config_layer('convt') +class ConvTransLayerBase(LayerBase): + layer_type = 'convt' + + def __init__(self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): + super(ConvTransLayerBase, self).__init__( + name, self.layer_type, 0, inputs=inputs, **xargs) + + if num_filters is not None: + self.config.num_filters = num_filters + + use_gpu = int(g_command_config_args.get("use_gpu", 0)) + parallel_nn = int(g_command_config_args.get("parallel_nn", 0)) + + # cudnn_convt has not been implemented so use exconvt only + self.layer_type = "exconvt" + # need to specify layer in config + self.config.type = self.layer_type + + if shared_biases is not None: + self.config.shared_biases = shared_biases + + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + parse_conv( + self.inputs[input_index].conv, + input_layer.name, + self.config.inputs[input_index].conv_conf, + num_filters, + trans=True) + conv_conf = self.config.inputs[input_index].conv_conf + psize = self.calc_parameter_size(conv_conf) + print("output size for %s is %d " % (name, conv_conf.output_x)) + self.create_input_parameter(input_index, psize) + self.set_layer_size( + (conv_conf.img_size**2) * self.config.num_filters) + + psize = self.config.size + if shared_biases: + psize = self.config.num_filters + self.create_bias_parameter(bias, psize, [psize, 1]) + + def calc_parameter_size(self, conv_conf): + return conv_conf.channels * conv_conf.filter_channels \ + * (conv_conf.filter_size * conv_conf.filter_size_y) + + +@config_layer('exconvt') +class ConvTransLayer(ConvTransLayerBase): + layer_type = 'exconvt' + + @config_layer('norm') class NormLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(NormLayer, self).__init__(name, 'norm', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(NormLayer, self).__init__( + name, 'norm', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_norm( - self.inputs[input_index].norm, - input_layer.name, - self.config.inputs[input_index].norm_conf) + parse_norm(self.inputs[input_index].norm, input_layer.name, + self.config.inputs[input_index].norm_conf) norm_conf = self.config.inputs[input_index].norm_conf - self.set_layer_size((norm_conf.output_x ** 2) * norm_conf.channels) + self.set_layer_size((norm_conf.output_x**2) * norm_conf.channels) + @config_layer('pool') class PoolLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(PoolLayer, self).__init__(name, 'pool', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(PoolLayer, self).__init__( + name, 'pool', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_pool( - self.inputs[input_index].pool, - input_layer.name, - self.config.inputs[input_index].pool_conf) + parse_pool(self.inputs[input_index].pool, input_layer.name, + self.config.inputs[input_index].pool_conf) pool_conf = self.config.inputs[input_index].pool_conf - print("output size for %s is %d*%d " % ( - name, pool_conf.output_y, pool_conf.output_x)) - self.set_layer_size((pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) + print("output size for %s is %d*%d " % (name, pool_conf.output_y, + pool_conf.output_x)) + self.set_layer_size( + (pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) + + +@config_layer('spp') +class SpatialPyramidPoolLayer(LayerBase): + def __init__(self, name, inputs, device=None): + super(SpatialPyramidPoolLayer, self).__init__( + name, 'spp', 0, inputs=inputs, device=device) + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + parse_spp(self.inputs[input_index].spp, input_layer.name, + self.config.inputs[input_index].spp_conf) + spp_conf = self.config.inputs[input_index].spp_conf + output_size = (pow(4, spp_conf.pyramid_height) - 1) / (4 - 1) + print("output size for %s is %d " % (name, output_size)) + self.set_layer_size(output_size * spp_conf.channels) + @config_layer('batch_norm') class BatchNormLayer(LayerBase): layer_type = 'batch_norm' - def __init__( - self, - name, - inputs, - active_type="linear", - bias=True, - device=None, - use_global_stats=True, - moving_average_fraction=0.9, - batch_norm_type=None, - **xargs): + + def __init__(self, + name, + inputs, + active_type="linear", + bias=True, + device=None, + use_global_stats=True, + moving_average_fraction=0.9, + batch_norm_type=None, + **xargs): if inputs is None: inputs = [] elif not isinstance(inputs, list): inputs = [inputs] - config_assert(len(inputs) == 1, - "BatchNormLayer must have one and only one input") + config_assert( + len(inputs) == 1, "BatchNormLayer must have one and only one input") # Create Input for moving mean and std, # in batch normalization layer. # These paras no need to update, so set is_static is true. @@ -1692,12 +1841,13 @@ class BatchNormLayer(LayerBase): use_gpu = bool(int(g_command_config_args.get("use_gpu", 0))) is_shared = True if not use_gpu else False for i in xrange(2): - inputs.append(Input(inputs[0].input_layer_name, - initial_std=0.0, - initial_mean=0.0, - is_static=True, - is_shared=is_shared, - )) + inputs.append( + Input( + inputs[0].input_layer_name, + initial_std=0.0, + initial_mean=0.0, + is_static=True, + is_shared=is_shared, )) parallel_nn = bool(int(g_command_config_args.get("parallel_nn", 0))) cudnn_version = int(g_command_config_args.get("cudnn_version", 0)) @@ -1707,21 +1857,25 @@ class BatchNormLayer(LayerBase): ((not parallel_nn) or self.config.device > -1) and \ cudnn_version >= 4007 self.layer_type = "cudnn_batch_norm" if use_cudnn else "batch_norm" - super(BatchNormLayer, self).__init__(name, self.layer_type, 0, - active_type=active_type, - inputs=inputs, device=device, **xargs) + super(BatchNormLayer, self).__init__( + name, + self.layer_type, + 0, + active_type=active_type, + inputs=inputs, + device=device, + **xargs) if use_global_stats is not None: self.config.use_global_stats = use_global_stats if moving_average_fraction is not None: self.config.moving_average_fraction = moving_average_fraction - input_layer= self.get_input_layer(0) - parse_image(self.inputs[0].image, - input_layer.name, + input_layer = self.get_input_layer(0) + parse_image(self.inputs[0].image, input_layer.name, self.config.inputs[0].image_conf) image_conf = self.config.inputs[0].image_conf - self.set_layer_size((image_conf.img_size ** 2) * image_conf.channels) + self.set_layer_size((image_conf.img_size**2) * image_conf.channels) psize = self.calc_parameter_size(image_conf) dims = [1, psize] @@ -1734,75 +1888,74 @@ class BatchNormLayer(LayerBase): def calc_parameter_size(self, image_conf): return image_conf.channels + @config_layer('trans') class TransLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(TransLayer, self).__init__(name, 'trans', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'TransLayer must have one and only one input') + def __init__(self, name, inputs, device=None): + super(TransLayer, self).__init__( + name, 'trans', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, + 'TransLayer must have one and only one input') self.set_layer_size(self.get_input_layer(0).size) + @config_layer('resize') class ResizeLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): - super(ResizeLayer, self).__init__(name, 'resize', size=size, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'ResizeLayer must have one and only one input') + def __init__(self, name, size, inputs, device=None): + super(ResizeLayer, self).__init__( + name, 'resize', size=size, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, + 'ResizeLayer must have one and only one input') + @config_layer('blockexpand') class BlockExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(BlockExpandLayer, self).__init__(name, 'blockexpand', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(BlockExpandLayer, self).__init__( + name, 'blockexpand', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_block_expand(self.inputs[input_index].block_expand, - input_layer.name, + parse_block_expand( + self.inputs[input_index].block_expand, input_layer.name, self.config.inputs[input_index].block_expand_conf) - block_expand_conf = self.config.inputs[input_index].block_expand_conf - self.set_layer_size(block_expand_conf.block_x * block_expand_conf.block_y - * block_expand_conf.channels) + block_expand_conf = self.config.inputs[ + input_index].block_expand_conf + self.set_layer_size(block_expand_conf.block_x * + block_expand_conf.block_y * + block_expand_conf.channels) + @config_layer('maxout') class MaxOutLayer(LayerBase): - def __init__( - self, - name, - inputs, - **xargs): - super(MaxOutLayer, self).__init__(name, 'maxout', 0, inputs=inputs, **xargs) + def __init__(self, name, inputs, **xargs): + super(MaxOutLayer, self).__init__( + name, 'maxout', 0, inputs=inputs, **xargs) input_layer = self.get_input_layer(0) - parse_maxout(self.inputs[0].maxout, - input_layer.name, + parse_maxout(self.inputs[0].maxout, input_layer.name, self.config.inputs[0].maxout_conf) maxout_conf = self.config.inputs[0].maxout_conf - self.set_layer_size(g_layer_map[input_layer.name].size / maxout_conf.groups) - + self.set_layer_size(g_layer_map[input_layer.name].size / + maxout_conf.groups) + + # key: cost type # value: cost class g_cost_map = {} + # define a cost layer without any parameters def define_cost(class_name, cost_type): def init(cls, name, inputs, device=None, coeff=1.): - super(type(cls), cls).__init__(name, cost_type, 1, inputs, device=device, coeff=coeff) + super(type(cls), cls).__init__( + name, cost_type, 1, inputs, device=device, coeff=coeff) - cls = type(class_name, (LayerBase,), dict(__init__=init)) + cls = type(class_name, (LayerBase, ), dict(__init__=init)) global g_cost_map g_cost_map[cost_type] = cls + define_cost('MultiClassCrossEntropy', 'multi-class-cross-entropy') define_cost('RankingCost', 'rank-cost') define_cost('AucValidation', 'auc-validation') @@ -1811,20 +1964,17 @@ define_cost('SumOfSquaresCostLayer', 'square_error') define_cost('MultiBinaryLabelCrossEntropy', 'multi_binary_label_cross_entropy') define_cost('SoftBinaryClassCrossEntropy', 'soft_binary_class_cross_entropy') define_cost('HuberTwoClass', 'huber') +define_cost('SumCost', 'sum_cost') + @config_layer('hsigmoid') class HierarchicalSigmoidLayer(LayerBase): - def __init__( - self, - name, - num_classes, - inputs, - device=None, - bias=True): + def __init__(self, name, num_classes, inputs, device=None, bias=True): super(HierarchicalSigmoidLayer, self).__init__( name, 'hsigmoid', 1, inputs=inputs, device=device) - config_assert(len(self.inputs) >= 2, - 'HierarchicalSigmoidLayer must have at least 2 inputs') + config_assert( + len(self.inputs) >= 2, + 'HierarchicalSigmoidLayer must have at least 2 inputs') self.config.num_classes = num_classes for input_index in xrange(len(self.inputs) - 1): input_layer = self.get_input_layer(input_index) @@ -1833,6 +1983,7 @@ class HierarchicalSigmoidLayer(LayerBase): self.create_input_parameter(input_index, psize, dims) self.create_bias_parameter(bias, num_classes - 1) + ''' lambdaCost for lambdaRank LTR approach @@ -1857,59 +2008,57 @@ Usage: max_sort_size can be greater than the size of a list, in which case the algorithm will sort the entire list to get gradient. ''' + + @config_layer('lambda_cost') class LambdaCost(LayerBase): - def __init__( - self, - name, - inputs, - NDCG_num = 5, - max_sort_size = -1, - device=None): + def __init__(self, name, inputs, NDCG_num=5, max_sort_size=-1, device=None): super(LambdaCost, self).__init__( name, 'lambda_cost', 1, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'lambdaCost must have 2 inputs') + config_assert(len(self.inputs) == 2, 'lambdaCost must have 2 inputs') self.config.NDCG_num = NDCG_num if max_sort_size != -1: - config_assert(NDCG_num <= max_sort_size, - 'NDCG_num must be less than or equal to max_sort_size') + config_assert( + NDCG_num <= max_sort_size, + 'NDCG_num must be less than or equal to max_sort_size') self.config.max_sort_size = max_sort_size + @config_layer('nce') class NCELayer(LayerBase): - def __init__( - self, - name, - num_classes, - inputs, - num_neg_samples=10, - neg_sampling_dist=None, - bias=True, - **xargs): + def __init__(self, + name, + num_classes, + inputs, + num_neg_samples=10, + neg_sampling_dist=None, + bias=True, + **xargs): super(NCELayer, self).__init__(name, 'nce', 1, inputs=inputs, **xargs) - config_assert(len(self.inputs) >= 2, - 'NCELayer must have at least 2 inputs') + config_assert( + len(self.inputs) >= 2, 'NCELayer must have at least 2 inputs') self.config.num_classes = num_classes if neg_sampling_dist is not None: - config_assert(len(neg_sampling_dist) == num_classes, - 'len(neg_sampling_dist)(%s) is not same as num_classes (%s)' - % (len(neg_sampling_dist), num_classes)) + config_assert( + len(neg_sampling_dist) == num_classes, + 'len(neg_sampling_dist)(%s) is not same as num_classes (%s)' % + (len(neg_sampling_dist), num_classes)) s = sum(neg_sampling_dist) - config_assert(abs(s - 1) < 1e-5, - 'The sum of neg_sampling_dist (%s) is not 1' % s) + config_assert( + abs(s - 1) < 1e-5, + 'The sum of neg_sampling_dist (%s) is not 1' % s) self.config.neg_sampling_dist.extend(neg_sampling_dist) self.config.num_neg_samples = num_neg_samples num_real_inputs = len(self.inputs) - 1 - input_layer = self.get_input_layer(num_real_inputs) + input_layer = self.get_input_layer(num_real_inputs) config_assert(input_layer.type == 'data', 'Expecting the last input layer of an nce layer to be ' 'a data layer') - if (num_real_inputs > 1 and input_layer.size == 1 - and self.get_input_layer(num_real_inputs - 1).type == 'data'): + if (num_real_inputs > 1 and input_layer.size == 1 and + self.get_input_layer(num_real_inputs - 1).type == 'data'): # This input layer is assumed to be a sample weight layer num_real_inputs -= 1 @@ -1923,105 +2072,82 @@ class NCELayer(LayerBase): @config_layer('addto') class AddToLayer(LayerBase): - def __init__( - self, - name, - inputs, - bias=True, - **xargs): + def __init__(self, name, inputs, bias=True, **xargs): super(AddToLayer, self).__init__( name, 'addto', 0, inputs=inputs, **xargs) - config_assert(len(inputs) > 0, - 'inputs cannot be empty for AddToLayer') + config_assert(len(inputs) > 0, 'inputs cannot be empty for AddToLayer') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('agent') class AgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): - super(AgentLayer, self).__init__(name, 'agent', size, inputs=[], device=device) + def __init__(self, name, size, device=None): + super(AgentLayer, self).__init__( + name, 'agent', size, inputs=[], device=device) + @config_layer('sequence_agent') class SequenceAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceAgentLayer, self).__init__( name, 'sequence_agent', size, inputs=[], device=device) + @config_layer('gather_agent') class GatherAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(GatherAgentLayer, self).__init__( name, 'gather_agent', size, inputs=[], device=device) + @config_layer('scatter_agent') class ScatterAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(ScatterAgentLayer, self).__init__( name, 'scatter_agent', size, inputs=[], device=device) + @config_layer('sequence_gather_agent') class SequenceGatherAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceGatherAgentLayer, self).__init__( - name, 'sequence_gather_agent', size, inputs=[], device=device) + name, 'sequence_gather_agent', size, inputs=[], device=device) + @config_layer('sequence_scatter_agent') class SequenceScatterAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceScatterAgentLayer, self).__init__( - name, 'sequence_scatter_agent', size, inputs=[], device=device) + name, 'sequence_scatter_agent', size, inputs=[], device=device) + @config_layer('multiplex') class MultiplexLayer(LayerBase): - def __init__( - self, - name, - inputs, - size, - device=None): - super(MultiplexLayer, self).__init__(name, 'multiplex', size, inputs=inputs, device=device) - config_assert(len(inputs) > 2, - 'MultiplexLayer should have more than 2 inputs.') + def __init__(self, name, inputs, size, device=None): + super(MultiplexLayer, self).__init__( + name, 'multiplex', size, inputs=inputs, device=device) + config_assert( + len(inputs) > 2, 'MultiplexLayer should have more than 2 inputs.') for i in range(1, len(inputs)): - config_assert(self.get_input_layer(i).size == size, - "All the input layers except the first one should" - "have the same size as the MultiplexLayer.") + config_assert( + self.get_input_layer(i).size == size, + "All the input layers except the first one should" + "have the same size as the MultiplexLayer.") + @config_func -def Link(name, - has_subseq=False, - ): +def Link( + name, + has_subseq=False, ): link_config = LinkConfig() link_config.link_name = name link_config.has_subseq = has_subseq return link_config + # memory for recurrent layer group. # *name* and *size* are actual layer's name and size. # will return name of the memory, @@ -2036,43 +2162,46 @@ def Link(name, # can only be initailized by a *boot_layer* which is a sequence. # @config_func -def Memory(name, - size, - is_sequence=False, - boot_layer=None, - boot_bias=False, - boot_bias_active_type="", - boot_with_const_id=None, - ): +def Memory( + name, + size, + is_sequence=False, + boot_layer=None, + boot_bias=False, + boot_bias_active_type="", + boot_with_const_id=None, ): agent_name = name + "+delay1" if is_sequence: agent_layer = SequenceAgentLayer(agent_name, size) else: agent_layer = AgentLayer(agent_name, size) config_assert(g_current_submodel.is_recurrent_layer_group, - 'Memory should be used in recurrent layer group only') + 'Memory should be used in recurrent layer group only') memory = g_current_submodel.memories.add() memory.layer_name = MakeLayerNameInSubmodel(name) memory.link_name = MakeLayerNameInSubmodel(agent_name) memory.is_sequence = is_sequence - options = sum((boot_layer is not None, - bool(boot_bias), + options = sum((boot_layer is not None, bool(boot_bias), boot_with_const_id is not None)) - config_assert(options <= 1, - 'take one option at most from boot_layer, boot_bias, or boot_with_const_id') + config_assert( + options <= 1, + 'take one option at most from boot_layer, boot_bias, or boot_with_const_id' + ) if boot_layer is not None: boot_layer = MakeLayerNameInParentSubmodel(boot_layer) config_assert(boot_layer in g_layer_map, - 'boot_layer "%s" does not correspond to a layer name' % boot_layer) + 'boot_layer "%s" does not correspond to a layer name' % + boot_layer) memory.boot_layer_name = boot_layer elif boot_bias: memory.boot_bias_parameter_name = agent_layer.create_bias_parameter( - boot_bias, size, for_self = False) + boot_bias, size, for_self=False) memory.boot_bias_active_type = boot_bias_active_type elif boot_with_const_id is not None: memory.boot_with_const_id = boot_with_const_id return agent_name + # Generator for recurrent layer group, to use it: # 1. define a id layer as output of layer group # 2. define a memory of this id layer, and assign a boot id(begin of sequence) @@ -2084,11 +2213,10 @@ def Memory(name, @config_func def Generator( max_num_frames, - eos_layer_name = "eos_check", - num_results_per_sample = 1, - beam_size = 1, - log_prob = None, - ): + eos_layer_name="eos_check", + num_results_per_sample=1, + beam_size=1, + log_prob=None, ): generator_config = GeneratorConfig() generator_config.max_num_frames = max_num_frames generator_config.eos_layer_name = eos_layer_name @@ -2098,60 +2226,55 @@ def Generator( generator_config.log_prob = log_prob return generator_config + @config_layer('expand') class ExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - trans_type='non-seq', - device=None, - bias=False): - super(ExpandLayer, self).__init__( - name, 'expand', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'ExpandLayer takes 2 and only 2 inputs') - self.config.trans_type = trans_type - for input_index in xrange(len(self.inputs)): - input_layer = self.get_input_layer(input_index) - self.set_layer_size(self.get_input_layer(0).size) - self.create_bias_parameter(bias, self.config.size) + def __init__(self, + name, + inputs, + trans_type='non-seq', + device=None, + bias=False): + super(ExpandLayer, self).__init__( + name, 'expand', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 2, 'ExpandLayer takes 2 and only 2 inputs') + self.config.trans_type = trans_type + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + self.set_layer_size(self.get_input_layer(0).size) + self.create_bias_parameter(bias, self.config.size) + @config_layer('featmap_expand') class FeatMapExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None, - num_filters=None, - bias=False): - super(FeatMapExpandLayer, self).__init__( - name, 'featmap_expand', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'ExpandLayer takes 1 and only 1 inputs') - if num_filters is not None: + def __init__(self, name, inputs, device=None, num_filters=None, bias=False): + super(FeatMapExpandLayer, self).__init__( + name, 'featmap_expand', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, 'ExpandLayer takes 1 and only 1 inputs') + if num_filters is not None: self.config.num_filters = num_filters - else: + else: logger.fatal("FeatMapExpandLayer must specify num_filters.") - self.set_layer_size(self.get_input_layer(0).size * num_filters) + self.set_layer_size(self.get_input_layer(0).size * num_filters) @config_layer('max') class MaxLayer(LayerBase): - def __init__( - self, - name, - inputs, - trans_type='non-seq', - active_type='linear', - device=None, - bias=False, - output_max_index=None): - super(MaxLayer, self).__init__(name, 'max', 0, inputs=inputs, device=device) + def __init__(self, + name, + inputs, + trans_type='non-seq', + active_type='linear', + device=None, + bias=False, + output_max_index=None): + super(MaxLayer, self).__init__( + name, 'max', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') - self.config.trans_type = trans_type - self.config.active_type = active_type + self.config.trans_type = trans_type + self.config.active_type = active_type for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2162,12 +2285,7 @@ class MaxLayer(LayerBase): @config_layer('maxid') class MaxIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - beam_size=None, - device=None): + def __init__(self, name, inputs, beam_size=None, device=None): super(MaxIdLayer, self).__init__( name, 'maxid', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxIdLayer must have 1 input') @@ -2185,37 +2303,39 @@ class MaxIdLayer(LayerBase): @config_layer('eos_id') class EosIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - eos_id, - device=None): + def __init__(self, name, inputs, eos_id, device=None): super(EosIdLayer, self).__init__( name, 'eos_id', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'EosIdLayer must have 1 input') - self.set_layer_size(2) # boolean output + self.set_layer_size(2) # boolean output self.config.eos_id = eos_id + @config_layer('seqlastins') class SequenceLastInstanceLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - trans_type='non-seq', - device=None, - bias=False): - super(SequenceLastInstanceLayer, self).__init__(name, 'seqlastins', - 0, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') - self.config.trans_type = trans_type + def __init__(self, + name, + inputs, + active_type='linear', + trans_type='non-seq', + device=None, + bias=False): + super(SequenceLastInstanceLayer, self).__init__( + name, + 'seqlastins', + 0, + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') + self.config.trans_type = trans_type for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('seqfirstins') class SequenceFirstInstanceLayer(SequenceLastInstanceLayer): def __init__( @@ -2225,167 +2345,163 @@ class SequenceFirstInstanceLayer(SequenceLastInstanceLayer): active_type='linear', trans_type='non-seq', device=None, - bias=False, - ): - super(SequenceFirstInstanceLayer, self).__init__(name, - inputs=inputs, active_type=active_type, device=device, bias=bias) - self.config.trans_type = trans_type + bias=False, ): + super(SequenceFirstInstanceLayer, self).__init__( + name, + inputs=inputs, + active_type=active_type, + device=device, + bias=bias) + self.config.trans_type = trans_type self.config.select_first = True + @config_layer('seqconcat') class SequenceConcatLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - device=None, - bias=False): - super(SequenceConcatLayer, self).__init__(name, 'seqconcat', - 0, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') + def __init__(self, + name, + inputs, + active_type='linear', + device=None, + bias=False): + super(SequenceConcatLayer, self).__init__( + name, + 'seqconcat', + 0, + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('seqreshape') class SequenceReshapeLayer(LayerBase): - def __init__( - self, - name, + def __init__(self, + name, + size, + inputs, + active_type='linear', + device=None, + bias=False): + super(SequenceReshapeLayer, self).__init__( + name, + 'seqreshape', size, - inputs, - active_type='linear', - device=None, - bias=False): - super(SequenceReshapeLayer, self).__init__(name, 'seqreshape', - size, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') self.set_layer_size(size) self.create_bias_parameter(bias, size) + @config_layer('subseq') class SubSequenceLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - device=None, - bias=False): - super(SubSequenceLayer, self).__init__(name, 'subseq', - 0, inputs=inputs, device=device, active_type=active_type) + def __init__(self, + name, + inputs, + active_type='linear', + device=None, + bias=False): + super(SubSequenceLayer, self).__init__( + name, + 'subseq', + 0, + inputs=inputs, + device=device, + active_type=active_type) config_assert(len(inputs) == 3, 'SubSequenceLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) size = input_layer0.size self.set_layer_size(size) self.create_bias_parameter(bias, size) + @config_layer('out_prod') class OuterProdLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(OuterProdLayer, self).__init__(name, 'out_prod', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(OuterProdLayer, self).__init__( + name, 'out_prod', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'OuterProdLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer0.size * input_layer1.size) + @config_layer('power') class PowerLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(PowerLayer, self).__init__(name, 'power', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(PowerLayer, self).__init__( + name, 'power', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'PowerLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) input_layer0 = self.get_input_layer(0) - config_assert(1==input_layer0.size, - 'The left input is the exponent and should be of size 1') + config_assert(1 == input_layer0.size, + 'The left input is the exponent and should be of size 1') + @config_layer('slope_intercept') class SlopeInterceptLayer(LayerBase): - def __init__( - self, - name, - inputs, - slope=1.0, - intercept=0.0, - device=None): - super(SlopeInterceptLayer, self).__init__(name, 'slope_intercept', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, slope=1.0, intercept=0.0, device=None): + super(SlopeInterceptLayer, self).__init__( + name, 'slope_intercept', 0, inputs=inputs, device=device) self.config.slope = slope self.config.intercept = intercept config_assert(len(inputs) == 1, 'SlopeInterceptLayer must have 1 input') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('scaling') class ScalingLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(ScalingLayer, self).__init__(name, 'scaling', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(ScalingLayer, self).__init__( + name, 'scaling', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ScalingLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) input_layer0 = self.get_input_layer(0) - config_assert(1==input_layer0.size, - 'The left input should be of size 1') + config_assert(1 == input_layer0.size, + 'The left input should be of size 1') + @config_layer('conv_shift') class ConvShiftLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(ConvShiftLayer, self).__init__(name, 'conv_shift', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(ConvShiftLayer, self).__init__( + name, 'conv_shift', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ConvShiftLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('convex_comb') class ConvexCombinationLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): + def __init__(self, name, size, inputs, device=None): super(ConvexCombinationLayer, self).__init__( - name, 'convex_comb', size, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'ConvexCombinationLayer must have 2 inputs') + name, 'convex_comb', size, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 2, 'ConvexCombinationLayer must have 2 inputs') config_assert( size * self.get_input_layer(0).size == self.get_input_layer(1).size, 'Wrong input size for ConvexCombinationLayer') self.set_layer_size(size) + @config_layer('interpolation') class InterpolationLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(InterpolationLayer, self).__init__( name, 'interpolation', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 3, - 'InterpolationLayer must have 3 inputs') + config_assert( + len(self.inputs) == 3, 'InterpolationLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) input_layer2 = self.get_input_layer(2) @@ -2394,64 +2510,51 @@ class InterpolationLayer(LayerBase): config_assert(input_layer1.size == input_layer2.size, 'the two vector inputs should be of the same size') + @config_layer('bilinear_interp') class BilinearInterpLayer(LayerBase): - def __init__( - self, - name, - inputs, - **xargs): + def __init__(self, name, inputs, **xargs): super(BilinearInterpLayer, self).__init__( name, 'bilinear_interp', 0, inputs=inputs, **xargs) input_layer = self.get_input_layer(0) - parse_bilinear(self.inputs[0].bilinear_interp, - input_layer.name, - self.config.inputs[0].bilinear_interp_conf); + parse_bilinear(self.inputs[0].bilinear_interp, input_layer.name, + self.config.inputs[0].bilinear_interp_conf) conf = self.inputs[0].bilinear_interp - self.set_layer_size(conf.out_size_x * conf.out_size_y * conf.num_channels) + self.set_layer_size(conf.out_size_x * conf.out_size_y * + conf.num_channels) + @config_layer('sum_to_one_norm') class SumToOneNormLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(SumToOneNormLayer, self).__init__( - name, 'sum_to_one_norm', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'SumToOneNormLayer must have 1 input') + name, 'sum_to_one_norm', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, 'SumToOneNormLayer must have 1 input') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('cos_vm') class CosSimVecMatLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - cos_scale=1.0, - device=None): + def __init__(self, name, size, inputs, cos_scale=1.0, device=None): super(CosSimVecMatLayer, self).__init__( - name, 'cos_vm', size, inputs=inputs, device=device) + name, 'cos_vm', size, inputs=inputs, device=device) self.config.cos_scale = cos_scale - config_assert(len(self.inputs) == 2, - 'CosSimVecMatLayer must have 2 inputs') + config_assert( + len(self.inputs) == 2, 'CosSimVecMatLayer must have 2 inputs') config_assert( size * self.get_input_layer(0).size == self.get_input_layer(1).size, 'Wrong input size for CosSimVecMatLayer') + @config_layer('sampling_id') class SamplingIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(SamplingIdLayer, self).__init__( name, 'sampling_id', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, 'SamplingIdLayer must have 1 input') + config_assert( + len(self.inputs) == 1, 'SamplingIdLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2464,33 +2567,33 @@ class SamplingIdLayer(LayerBase): # 'squarerootn': sum each sample, but divide by sqrt(sample_num). @config_layer('average') class AverageLayer(LayerBase): - def __init__( - self, - name, - inputs, - average_strategy='average', - trans_type='non-seq', - active_type='linear', - device=None, - bias=False): - super(AverageLayer, self).__init__(name, 'average', 0, inputs=inputs, - device=device, active_type=active_type) + def __init__(self, + name, + inputs, + average_strategy='average', + trans_type='non-seq', + active_type='linear', + device=None, + bias=False): + super(AverageLayer, self).__init__( + name, + 'average', + 0, + inputs=inputs, + device=device, + active_type=active_type) self.config.average_strategy = average_strategy - self.config.trans_type = trans_type + self.config.trans_type = trans_type config_assert(len(inputs) == 1, 'AverageLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('cos') class CosSimLayer(LayerBase): - def __init__( - self, - name, - inputs, - cos_scale=5, - device=None): + def __init__(self, name, inputs, cos_scale=5, device=None): super(CosSimLayer, self).__init__( name, 'cos', 1, inputs=inputs, device=device) config_assert(len(self.inputs) == 2, 'CosSimLayer must have 2 inputs') @@ -2502,18 +2605,13 @@ class CosSimLayer(LayerBase): @config_layer('tensor') class TensorLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None, - bias=True, - **xargs): - super(TensorLayer, self).__init__(name, 'tensor', size, inputs=inputs, device=device, **xargs) + def __init__(self, name, size, inputs, device=None, bias=True, **xargs): + super(TensorLayer, self).__init__( + name, 'tensor', size, inputs=inputs, device=device, **xargs) config_assert(len(self.inputs) == 2, 'TensorLayer must have 2 inputs') config_assert(size > 0, 'size must be positive') - config_assert(inputs[1].parameter_name == None, 'second parameter should be None.') + config_assert(inputs[1].parameter_name == None, + 'second parameter should be None.') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) psize = size * input_layer0.size * input_layer1.size @@ -2524,14 +2622,13 @@ class TensorLayer(LayerBase): @config_layer('mixed') class MixedLayer(LayerBase): - def __init__( - self, - name, - inputs, - size=0, - bias=True, - error_clipping_threshold=None, - **xargs): + def __init__(self, + name, + inputs, + size=0, + bias=True, + error_clipping_threshold=None, + **xargs): config_assert(inputs, 'inputs cannot be empty') super(MixedLayer, self).__init__( name, 'mixed', size, inputs=inputs, **xargs) @@ -2556,24 +2653,28 @@ class MixedLayer(LayerBase): else: sz = operator.calc_output_size(operator_conf.input_sizes) if sz != 0: - config_assert(sz == self.config.size, - "different inputs have different size: %s vs. %s" % - (sz, self.config.size)) + config_assert( + sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] if input_index not in operator_input_index: - config_assert(isinstance(input, Projection), "input should be projection or operation") + config_assert( + isinstance(input, Projection), + "input should be projection or operation") if self.config.size == 0 and isinstance(input, Projection): size = input.calc_output_size(input_layer) if size != 0: self.set_layer_size(size) elif isinstance(input, Projection): - sz = input.calc_output_size(input_layer) - if sz != 0: - config_assert(sz == self.config.size, - "different inputs have different size: %s vs. %s" % - (sz, self.config.size)) + sz = input.calc_output_size(input_layer) + if sz != 0: + config_assert( + sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) config_assert(size != 0, "size is not set") for input_index in xrange(len(self.inputs)): @@ -2585,7 +2686,8 @@ class MixedLayer(LayerBase): input_config = self.config.inputs[input_index] input_config.proj_conf.CopyFrom(input.proj_conf) - input_config.proj_conf.name = gen_parameter_name(name, input_index) + input_config.proj_conf.name = gen_parameter_name(name, + input_index) psize = input.calc_parameter_size(input_layer.size, size) dims = input.calc_parameter_dims(input_layer.size, size) self.create_input_parameter(input_index, psize, dims) @@ -2611,21 +2713,16 @@ class MixedLayer(LayerBase): if error_clipping_threshold is not None: self.config.error_clipping_threshold = error_clipping_threshold + # like MixedLayer, but no bias parameter @config_func -def ExpressionLayer(name, - inputs, - **xargs): +def ExpressionLayer(name, inputs, **xargs): MixedLayer(name, inputs, bias=False, **xargs) + @config_layer('concat') class ConcatenateLayer(LayerBase): - def __init__( - self, - name, - inputs, - bias=False, - **xargs): + def __init__(self, name, inputs, bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') config_assert(not bias, 'ConcatenateLayer cannot support bias.') super(ConcatenateLayer, self).__init__( @@ -2634,30 +2731,27 @@ class ConcatenateLayer(LayerBase): for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] - if self.config.size == 0: + if self.config.size == 0: size += input_layer.size self.set_layer_size(size) + # like concat layer, but each input layer was processed by a Projection. @config_layer('concat2') class ConcatenateLayer2(LayerBase): - def __init__( - self, - name, - inputs, - bias=False, - **xargs): + def __init__(self, name, inputs, bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') super(ConcatenateLayer2, self).__init__( name, 'concat2', 0, inputs=inputs, **xargs) if isinstance(self.inputs[0], ConvProjection): - for input_index in xrange(len(self.inputs) - 1): - input = self.inputs[input_index + 1] - config_assert(isinstance(input, ConvProjection), - "The first input of ConcatenateLayer2 is ConvProjection, " - "the other inputs should also be ConvProjection.") + for input_index in xrange(len(self.inputs) - 1): + input = self.inputs[input_index + 1] + config_assert( + isinstance(input, ConvProjection), + "The first input of ConcatenateLayer2 is ConvProjection, " + "the other inputs should also be ConvProjection.") size = 0 for input_index in xrange(len(self.inputs)): @@ -2679,9 +2773,9 @@ class ConcatenateLayer2(LayerBase): input_config.proj_conf.CopyFrom(input.proj_conf) input_config.proj_conf.name = gen_parameter_name(name, input_index) psize = input.calc_parameter_size(input.proj_conf.input_size, - input.proj_conf.output_size) + input.proj_conf.output_size) dims = input.calc_parameter_dims(input.proj_conf.input_size, - input.proj_conf.output_size) + input.proj_conf.output_size) self.create_input_parameter(input_index, psize, dims) psize = self.config.size @@ -2695,16 +2789,12 @@ class ConcatenateLayer2(LayerBase): self.config.bias_size = psize self.create_bias_parameter(bias, psize) + @config_layer('recurrent') class RecurrentLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - bias=True, - **xargs): - super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, **xargs) + def __init__(self, name, inputs, reversed=False, bias=True, **xargs): + super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, + **xargs) config_assert(len(self.inputs) == 1, 'RecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) size = input_layer.size @@ -2714,17 +2804,17 @@ class RecurrentLayer(LayerBase): self.create_input_parameter(0, size * size, dims) self.create_bias_parameter(bias, self.config.size) + @config_layer('lstmemory') class LstmLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): + def __init__(self, + name, + inputs, + reversed=False, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): super(LstmLayer, self).__init__(name, 'lstmemory', 0, inputs, **xargs) config_assert(len(self.inputs) == 1, 'LstmLayer must have 1 input') input_layer = self.get_input_layer(0) @@ -2733,117 +2823,126 @@ class LstmLayer(LayerBase): size = input_layer.size / 4 self.set_layer_size(size) self.config.reversed = reversed - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type self.create_input_parameter(0, size * size * 4, [size, size, 4]) #bias includes 3 kinds of peephole, 4 + 3 = 7 self.create_bias_parameter(bias, size * 7) + @config_layer('lstm_step') class LstmStepLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): - super(LstmStepLayer, self).__init__(name, 'lstm_step', - size, inputs, **xargs) + def __init__(self, + name, + size, + inputs, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): + super(LstmStepLayer, self).__init__(name, 'lstm_step', size, inputs, + **xargs) config_assert(len(inputs) == 2, 'LstmStepLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) - config_assert(input_layer0.size == 4 * size, 'input_layer0.size != 4 * layer.size') - config_assert(input_layer1.size == size, 'input_layer1.size != layer.size') - self.config.active_gate_type = active_gate_type + config_assert(input_layer0.size == 4 * size, + 'input_layer0.size != 4 * layer.size') + config_assert(input_layer1.size == size, + 'input_layer1.size != layer.size') + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type self.create_bias_parameter(bias, size * 3) + # get the specific output from the input layer. @config_layer('get_output') class GetOutputLayer(LayerBase): - def __init__( - self, - name, - size, - inputs): - super(GetOutputLayer, self).__init__(name, 'get_output' , size, inputs) - config_assert(len(self.inputs) == 1, 'GetOutputLayer must have 1 inputs') + def __init__(self, name, size, inputs): + super(GetOutputLayer, self).__init__(name, 'get_output', size, inputs) + config_assert( + len(self.inputs) == 1, 'GetOutputLayer must have 1 inputs') inputs = self.inputs[0] config_assert(inputs.input_layer_argument, 'input_layer_argument cannot be empty') + @config_layer('mdlstmemory') class MDLstmLayer(LayerBase): - def __init__( - self, - name, - inputs, - directions=True, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): - super(MDLstmLayer, self).__init__(name, 'mdlstmemory', 0, inputs, **xargs) + def __init__(self, + name, + inputs, + directions=True, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): + super(MDLstmLayer, self).__init__(name, 'mdlstmemory', 0, inputs, + **xargs) config_assert(len(self.inputs) == 1, 'MDLstmLayer must have 1 input') input_layer = self.get_input_layer(0) dim_num = len(directions) #check input_layer.size is divided by (3+dim_num) - config_assert(input_layer.size % (3+dim_num) == 0, "size % (dim_num) should be 0!") - size = input_layer.size / (3+dim_num) + config_assert(input_layer.size % (3 + dim_num) == 0, + "size % (dim_num) should be 0!") + size = input_layer.size / (3 + dim_num) self.set_layer_size(size) - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type for i in xrange(len(directions)): self.config.directions.append(int(directions[i])) - self.create_input_parameter(0, size * size * (3+dim_num), [size, size, 3+dim_num]) + self.create_input_parameter(0, size * size * (3 + dim_num), + [size, size, 3 + dim_num]) #bias includes 3 kinds of peephole, 3+dim_num+2+dim_num - self.create_bias_parameter(bias, size * (5+2*dim_num)) + self.create_bias_parameter(bias, size * (5 + 2 * dim_num)) + @config_layer('gated_recurrent') class GatedRecurrentLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - active_gate_type="sigmoid", - bias=True, - **xargs): - super(GatedRecurrentLayer, self).__init__(name, 'gated_recurrent', 0, inputs, **xargs) - config_assert(len(self.inputs) == 1, 'GatedRecurrentLayer must have 1 input') + def __init__(self, + name, + inputs, + reversed=False, + active_gate_type="sigmoid", + bias=True, + **xargs): + super(GatedRecurrentLayer, self).__init__(name, 'gated_recurrent', 0, + inputs, **xargs) + config_assert( + len(self.inputs) == 1, 'GatedRecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) #check input_layer.size is divided by 3 config_assert(input_layer.size % 3 == 0, "size % 3 should be 0!") size = input_layer.size / 3 self.set_layer_size(size) self.config.reversed = reversed - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.create_input_parameter(0, size * size * 3, [size, size * 3]) self.create_bias_parameter(bias, size * 3) + @config_layer('gru_step') class GruStepLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - active_gate_type="sigmoid", - bias=True, - **xargs): - super(GruStepLayer, self).__init__(name, 'gru_step', size, inputs, **xargs) + def __init__(self, + name, + size, + inputs, + active_gate_type="sigmoid", + bias=True, + **xargs): + super(GruStepLayer, self).__init__(name, 'gru_step', size, inputs, + **xargs) config_assert(len(self.inputs) == 2, 'GruStepLayer must have 2 input') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) - config_assert(input_layer0.size == 3 * size, 'input_layer0.size != 3 * layer.size') - config_assert(input_layer1.size == size, 'input_layer1.size != layer.size') - self.config.active_gate_type = active_gate_type + config_assert(input_layer0.size == 3 * size, + 'input_layer0.size != 3 * layer.size') + config_assert(input_layer1.size == size, + 'input_layer1.size != layer.size') + self.config.active_gate_type = active_gate_type self.create_input_parameter(0, size * size * 3, [size, size * 3]) self.create_bias_parameter(bias, size * 3) + ''' A layer for calculating the cost of sequential conditional random field model. Example: CRFLayer(name="crf_cost", size=label_num, @@ -2851,20 +2950,18 @@ class GruStepLayer(LayerBase): where "weight" is optional, one weight for each sequence @param coeff: weight of the layer ''' + + @config_layer('crf') class CRFLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - coeff=1.0, - device=None): + def __init__(self, name, size, inputs, coeff=1.0, device=None): super(CRFLayer, self).__init__(name, 'crf', size, inputs, device=device) - config_assert(2 <= len(self.inputs) <= 3, 'CRFLayer must have 2 or 3 inputs') + config_assert(2 <= len(self.inputs) <= 3, + 'CRFLayer must have 2 or 3 inputs') self.create_input_parameter(0, size * (size + 2), [size, size + 2]) self.config.coeff = coeff + ''' A layer for calculating the decoding sequence of sequential conditional random field model. @@ -2873,14 +2970,11 @@ class CRFLayer(LayerBase): this layer will also calculate error, output_.value[i] is 1 for incorrect decoding or 0 for correct decoding ''' + + @config_layer('crf_decoding') class CRFDecodingLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): + def __init__(self, name, size, inputs, device=None): super(CRFDecodingLayer, self).__init__( name, 'crf_decoding', size, inputs, device=device) config_assert( @@ -2888,47 +2982,35 @@ class CRFDecodingLayer(LayerBase): 'CRFDecodingLayer cannot have more than 2 inputs') self.create_input_parameter(0, size * (size + 2), [size, size + 2]) + @config_layer('ctc') class CTCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - norm_by_times = False, - device=None): + def __init__(self, name, size, inputs, norm_by_times=False, device=None): super(CTCLayer, self).__init__(name, 'ctc', size, inputs, device=device) self.config.norm_by_times = norm_by_times config_assert(len(self.inputs) == 2, 'CTCLayer must have 2 inputs') + @config_layer('recurrent_layer_group') class RecurrentLayerGroup(LayerBase): - def __init__( - self, - name, - device=None): + def __init__(self, name, device=None): super(RecurrentLayerGroup, self).__init__( name, 'recurrent_layer_group', 0, inputs=[], device=device) # Deprecated, use a new layer specific class instead @config_func -def Layer( - name, - type, - **xargs): +def Layer(name, type, **xargs): layers = {} layers.update(g_cost_map) layers.update(g_layer_type_map) layer_func = layers.get(type) - config_assert(layer_func, - "layer type '%s' not supported." % type) - layer_func(name, **xargs) + config_assert(layer_func, "layer type '%s' not supported." % type) + return layer_func(name, **xargs) + @config_func -def ParameterHook( - type, - **kwargs): +def ParameterHook(type, **kwargs): if type == 'pruning': mask_filename = kwargs.get('mask_filename', None) assert mask_filename is not None @@ -2941,30 +3023,28 @@ def ParameterHook( @config_func -def Parameter( - name, - size, - device, - dims, - learning_rate=None, - momentum=None, - decay_rate=None, - decay_rate_l1=None, - initial_mean=None, - initial_std=None, - initial_strategy=None, - initial_smart=None, - num_batches_regularization=None, - sparse_remote_update=None, - sparse_update=None, - gradient_clipping_threshold=None, - sparse=None, - format=None, - need_compact=None, - is_static=None, - is_shared=None, - update_hooks=None - ): +def Parameter(name, + size, + device, + dims, + learning_rate=None, + momentum=None, + decay_rate=None, + decay_rate_l1=None, + initial_mean=None, + initial_std=None, + initial_strategy=None, + initial_smart=None, + num_batches_regularization=None, + sparse_remote_update=None, + sparse_update=None, + gradient_clipping_threshold=None, + sparse=None, + format=None, + need_compact=None, + is_static=None, + is_shared=None, + update_hooks=None): config_assert(name not in g_parameter_map, 'Duplicated parameter name: ' + name) @@ -2995,8 +3075,8 @@ def Parameter( para.initial_std = default(initial_std, g_default_initial_std) para.initial_mean = default(initial_mean, g_default_initial_mean) - num_batches_regularization = default( - num_batches_regularization, g_default_num_batches_regularization) + num_batches_regularization = default(num_batches_regularization, + g_default_num_batches_regularization) if num_batches_regularization is not None: para.num_batches_regularization = int(num_batches_regularization) @@ -3006,18 +3086,21 @@ def Parameter( g_config.opt_config.use_sparse_remote_updater = True if sparse_update is not None: para.sparse_update = sparse_update - gradient_clipping_threshold = default( - gradient_clipping_threshold, g_default_gradient_clipping_threshold) + gradient_clipping_threshold = default(gradient_clipping_threshold, + g_default_gradient_clipping_threshold) if gradient_clipping_threshold is not None: para.gradient_clipping_threshold = gradient_clipping_threshold - para.initial_strategy = default(initial_strategy, g_default_initial_strategy) + para.initial_strategy = default(initial_strategy, + g_default_initial_strategy) para.initial_smart = default(initial_smart, g_default_initial_smart) if para.initial_smart: para.initial_mean = 0. if len(para.dims) != 0: para.initial_std = 1. / math.sqrt(para.dims[0]) else: - print("Use initial_smart, but dims not set. Initial_smart may not be used in this layer") + print( + "Use initial_smart, but dims not set. Initial_smart may not be used in this layer" + ) traceback.print_exc() para.initial_std = 1. / math.sqrt(para.size) if g_default_compact_func is not None: @@ -3056,64 +3139,78 @@ def default_initial_std(val): global g_default_initial_std g_default_initial_std = val + @config_func def default_initial_mean(val): global g_default_initial_mean g_default_initial_mean = val + @config_func def default_initial_strategy(val): global g_default_initial_strategy g_default_initial_strategy = val + @config_func def default_initial_smart(val): global g_default_initial_smart g_default_initial_smart = val + @config_func def default_momentum(val): global g_default_momentum g_default_momentum = val + @config_func def default_decay_rate(val): global g_default_decay_rate g_default_decay_rate = val + @config_func def default_num_batches_regularization(val): global g_default_num_batches_regularization g_default_num_batches_regularization = val + @config_func def default_gradient_clipping_threshold(val): global g_default_gradient_clipping_threshold g_default_gradient_clipping_threshold = val + @config_func def default_device(val): global g_default_device g_default_device = val + @config_func def default_update_hooks(val): global g_default_update_hooks g_default_update_hooks = val + @config_func def default_compact_func(val): global g_default_compact_func g_default_compact_func = val + def make_importer(config_dir, config_args): def Import(config_file, local_args={}): if not config_file.startswith('/'): config_file = config_dir + '/' + config_file g_config.config_files.append(config_file) - execfile(config_file, make_config_environment(config_file, config_args), local_args) + execfile(config_file, + make_config_environment(config_file, config_args), local_args) + return Import + settings = dict( batch_size=None, mini_batch_size=None, @@ -3142,26 +3239,24 @@ settings = dict( ada_rou=0.95, delta_add_rate=1.0, shrink_parameter_value=0, - adam_beta1 = 0.9, - adam_beta2 = 0.999, - adam_epsilon = 1e-8, -) + adam_beta1=0.9, + adam_beta2=0.999, + adam_epsilon=1e-8, ) -settings_deprecated = dict( - usage_ratio=1., -) +settings_deprecated = dict(usage_ratio=1., ) trainer_settings = dict( save_dir="./output/model", init_model_path=None, - start_pass=0, -) + start_pass=0, ) + @config_func def Settings(**args): for k, v in args.iteritems(): if k == "usage_ratio": - logger.warning("Deprecated: define usage_ratio in DataConfig instead") + logger.warning( + "Deprecated: define usage_ratio in DataConfig instead") if g_config.HasField("data_config"): g_config.data_config.__setattr__(k, v) settings_deprecated[k] = v @@ -3173,10 +3268,12 @@ def Settings(**args): else: logger.fatal('Unkown setting: %s' % k) + @config_func def cluster_config(**args): pass + @config_func def EnableSubmodelSuffix(flag=True): """ @@ -3186,10 +3283,12 @@ def EnableSubmodelSuffix(flag=True): global g_add_submodel_suffix g_add_submodel_suffix = flag + def make_config_environment(config_file, config_args): def make_setter(k): def setter(v): logger.fatal("Obsolete: use Settings(%s=%s, ...) instead" % (k, v)) + return setter funcs = {} @@ -3205,13 +3304,13 @@ def make_config_environment(config_file, config_args): funcs.update( Import=make_importer(config_dir, config_args), - get_config_arg=make_get_config_arg(config_args), - ) + get_config_arg=make_get_config_arg(config_args), ) funcs.update(g_extended_config_funcs) return funcs + def make_get_config_arg(config_args): def get_config_arg(name, type, default=None): if type == bool: @@ -3228,6 +3327,7 @@ def make_get_config_arg(config_args): return get_config_arg + def importlib(name): __import__(name) return sys.modules[name] @@ -3240,10 +3340,12 @@ def find_caller(): return s[0], s[1], s[2] return "(unknown file)", 0, "(unknown function)" + def my_fatal(s): logger.critical(s) raise Exception() + def parse_config(config_file, config_arg_str): ''' @param config_arg_str: a string of the form var1=val1,var2=val2. It will be @@ -3281,7 +3383,7 @@ def parse_config(config_file, config_arg_str): for k, v in settings.iteritems(): if v is None: continue - g_config.opt_config.__setattr__(k, v); + g_config.opt_config.__setattr__(k, v) for k, v in trainer_settings.iteritems(): if v is None: @@ -3308,6 +3410,7 @@ def parse_config_and_serialize(config_file, config_arg_str): traceback.print_exc() raise + if __name__ == '__main__': try: config = parse_config(sys.argv[1], '') diff --git a/python/paddle/trainer/config_parser_extension.py b/python/paddle/trainer/config_parser_extension.py index 3445076274b0a87254c6af6c3417fe51022c7891..ba4c79efdc10ec6cc895e76ddb87bc3fbd19ddc1 100644 --- a/python/paddle/trainer/config_parser_extension.py +++ b/python/paddle/trainer/config_parser_extension.py @@ -17,11 +17,10 @@ from paddle.proto.DataConfig_pb2 import DataConfig g_config = None -def SimpleData( - files=None, - feat_dim=None, - context_len=None, - buffer_capacity=None): +def SimpleData(files=None, + feat_dim=None, + context_len=None, + buffer_capacity=None): data_config = DataConfig() data_config.type = 'simple' @@ -33,6 +32,7 @@ def SimpleData( data_config.buffer_capacity = buffer_capacity return data_config + def get_config_funcs(trainer_config): global g_config g_config = trainer_config diff --git a/python/paddle/trainer/recurrent_units.py b/python/paddle/trainer/recurrent_units.py index 7d51de78b0d79ce59bd08507660ebe99b273a4a0..a80ad13d1ed52d84c3b5882939271b91ecc07bb3 100644 --- a/python/paddle/trainer/recurrent_units.py +++ b/python/paddle/trainer/recurrent_units.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - + # recurrent_units.py # Version 2.0 # @@ -22,161 +22,175 @@ from paddle.trainer.config_parser import * + # long short term memory, can be used in recurrent machine # *inputs* must be a list of Projections, for example: # inputs = [FullMatrixProjection("input_layer_name")], # *para_prefix* defines parameter names, if the *para_prefix* of # two LstmRecurrentUnit is same, they share same parameters # *out_memory* can be defined outside if it's used outside -def LstmRecurrentUnit(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): +def LstmRecurrentUnit(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): - if para_prefix is None: + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) + + state_memory = Memory(name=name + "_" + "state", size=size) - state_memory = Memory(name = name + "_" + "state", size = size) - Layer( - name = name + "_" + "input_recurrent", - type = "mixed", - size = size * 4, #(input_s, input_gate, forget_gate, output_gate) - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, - parameter_name = para_prefix + "_input_recurrent.b"), - inputs = inputs + [ - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_input_recurrent.w"), - ], - ) + name=name + "_" + "input_recurrent", + type="mixed", + size=size * 4, #(input_s, input_gate, forget_gate, output_gate) + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_input_recurrent.b"), + inputs=inputs + [ + FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_input_recurrent.w"), + ], ) LstmStepLayer( - name = name, - size = size, - bias = Bias(parameter_name = para_prefix + "_check.b"), - inputs = [name + "_" + "input_recurrent", state_memory], - active_type = active_type, - active_gate_type = gate_active_type, - active_state_type = state_active_type, - ) + name=name, + size=size, + bias=Bias(parameter_name=para_prefix + "_check.b"), + inputs=[name + "_" + "input_recurrent", state_memory], + active_type=active_type, + active_gate_type=gate_active_type, + active_state_type=state_active_type, ) GetOutputLayer( - name = name + "_" + "state", - size = size, - inputs = Input(name, input_layer_argument = "state"), - ) - -def LstmRecurrentUnitNaive(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - - if para_prefix is None: + name=name + "_" + "state", + size=size, + inputs=Input( + name, input_layer_argument="state"), ) + + +def LstmRecurrentUnitNaive(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) + + state_memory = Memory(name=name + "_" + "state", size=size) - state_memory = Memory(name = name + "_" + "state", size = size) - Layer( - name = name + "_" + "input_recurrent", - type = "mixed", - size = size * 4, #(input_s, input_gate, forget_gate, output_gate) - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, - parameter_name = para_prefix + "_input_recurrent.b"), - inputs = inputs + [ - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_input_recurrent.w"), - ], - ) + name=name + "_" + "input_recurrent", + type="mixed", + size=size * 4, #(input_s, input_gate, forget_gate, output_gate) + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_input_recurrent.b"), + inputs=inputs + [ + FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_input_recurrent.w"), + ], ) ExpressionLayer( - name = name + "_" + "input_s", - size = size, - active_type = active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=0)], - ) + name=name + "_" + "input_s", + size=size, + active_type=active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=0) + ], ) ExpressionLayer( - name = name + "_" + "input_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size), - DotMulProjection(state_memory, - parameter_name = para_prefix + "_input_check.w")], - ) + name=name + "_" + "input_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size), DotMulProjection( + state_memory, parameter_name=para_prefix + "_input_check.w") + ], ) ExpressionLayer( - name = name + "_" + "forget_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size*2), - DotMulProjection(state_memory, - parameter_name = para_prefix + "_forget_check.w")], - ) + name=name + "_" + "forget_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size * 2), + DotMulProjection( + state_memory, parameter_name=para_prefix + "_forget_check.w") + ], ) ExpressionLayer( - name = name + "_" + "state", - inputs = [DotMulOperator([name + "_" + "input_s", - name + "_" + "input_gate"]), - DotMulOperator([state_memory, - name + "_" + "forget_gate"]), - ], - ) + name=name + "_" + "state", + inputs=[ + DotMulOperator([name + "_" + "input_s", name + "_" + "input_gate"]), + DotMulOperator([state_memory, name + "_" + "forget_gate"]), + ], ) ExpressionLayer( - name = name + "_" + "output_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size*3), - DotMulProjection(name + "_" + "state", - parameter_name = para_prefix + "_output_check.w")], - ) + name=name + "_" + "output_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size * 3), + DotMulProjection( + name + "_" + "state", + parameter_name=para_prefix + "_output_check.w") + ], ) ExpressionLayer( - name = name + "_" + "state_atv", - active_type = state_active_type, - inputs = IdentityProjection(name + "_" + "state"), - ) + name=name + "_" + "state_atv", + active_type=state_active_type, + inputs=IdentityProjection(name + "_" + "state"), ) ExpressionLayer( - name = name, - inputs = DotMulOperator([name + "_" + "state_atv", - name + "_" + "output_gate"]), - ) + name=name, + inputs=DotMulOperator( + [name + "_" + "state_atv", name + "_" + "output_gate"]), ) + # like LstmRecurrentUnit, but it's a layer group. # it is equivalent to LstmLayer -def LstmRecurrentLayerGroup(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - seq_reversed = False): +def LstmRecurrentLayerGroup(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + seq_reversed=False): input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 4, - active_type = "", - bias = False, - inputs = inputs, - ) - - RecurrentLayerGroupBegin(name + "_layer_group", - in_links = [input_layer_name], - out_links = [name], - seq_reversed = seq_reversed) + name=input_layer_name, + type="mixed", + size=size * 4, + active_type="", + bias=False, + inputs=inputs, ) + + RecurrentLayerGroupBegin( + name + "_layer_group", + in_links=[input_layer_name], + out_links=[name], + seq_reversed=seq_reversed) LstmRecurrentUnit( - name = name, - size = size, - active_type = active_type, - state_active_type = state_active_type, - gate_active_type = gate_active_type, - inputs = [IdentityProjection(input_layer_name)], - para_prefix = para_prefix, - error_clipping_threshold = error_clipping_threshold, - ) + name=name, + size=size, + active_type=active_type, + state_active_type=state_active_type, + gate_active_type=gate_active_type, + inputs=[IdentityProjection(input_layer_name)], + para_prefix=para_prefix, + error_clipping_threshold=error_clipping_threshold, ) RecurrentLayerGroupEnd(name + "_layer_group") - # gated recurrent unit, can be used in recurrent machine # *inputs* should be a list of Projections, for example: # inputs = [FullMatrixProjection("input_layer_name")], @@ -184,142 +198,157 @@ def LstmRecurrentLayerGroup(name, size, # two GatedRecurrentUnit is same, they share same parameters # *out_memory* can be defined outside if it's used outside -def GatedRecurrentUnit(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup + +def GatedRecurrentUnit(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup input_layer_name = inputs else: input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - if para_prefix is None: + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) GruStepLayer( - name = name, - size = size, - bias = Bias(parameter_name = para_prefix + "_gate.b"), - inputs = [input_layer_name, - Input(out_memory, parameter_name = para_prefix + "_gate.w")], - active_type = active_type, - active_gate_type = gate_active_type, - ) - -def GatedRecurrentUnitNaive(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - - if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup + name=name, + size=size, + bias=Bias(parameter_name=para_prefix + "_gate.b"), + inputs=[ + input_layer_name, Input( + out_memory, parameter_name=para_prefix + "_gate.w") + ], + active_type=active_type, + active_gate_type=gate_active_type, ) + + +def GatedRecurrentUnitNaive(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + + if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup input_layer_name = inputs else: input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - if para_prefix is None: + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) Layer( - name = name + "_" + "update_gate", - type = "mixed", - size = size, - active_type = gate_active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_update_gate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=0), - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_update_gate.w")], - ) + name=name + "_" + "update_gate", + type="mixed", + size=size, + active_type=gate_active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_update_gate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=0), FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_update_gate.w") + ], ) Layer( - name = name + "_" + "reset_gate", - type = "mixed", - size = size, - active_type = gate_active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_reset_gate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=size), - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_reset_gate.w")], - ) + name=name + "_" + "reset_gate", + type="mixed", + size=size, + active_type=gate_active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_reset_gate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=size), FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_reset_gate.w") + ], ) ExpressionLayer( - name = name + "_" + "reset_output", - inputs = DotMulOperator([out_memory, name + "_" + "reset_gate"]), - ) + name=name + "_" + "reset_output", + inputs=DotMulOperator([out_memory, name + "_" + "reset_gate"]), ) Layer( - name = name + "_" + "output_candidate", - type = "mixed", - size = size, - active_type = active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_output_candidate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=size*2), - FullMatrixProjection(name + "_" + "reset_output", - parameter_name = para_prefix + "_output_candidate.w")], - ) - ExpressionLayer( #element-wise interpolation - name = name, - inputs = [IdentityProjection(out_memory), - DotMulOperator([out_memory, - name + "_" + "update_gate"], scale=-1.0), - DotMulOperator([name + "_" + "output_candidate", - name + "_" + "update_gate"]), - ], - ) + name=name + "_" + "output_candidate", + type="mixed", + size=size, + active_type=active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_output_candidate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=size * 2), FullMatrixProjection( + name + "_" + "reset_output", + parameter_name=para_prefix + "_output_candidate.w") + ], ) + ExpressionLayer( #element-wise interpolation + name=name, + inputs=[ + IdentityProjection(out_memory), + DotMulOperator( + [out_memory, name + "_" + "update_gate"], scale=-1.0), + DotMulOperator( + [name + "_" + "output_candidate", name + "_" + "update_gate"]), + ], ) + # like GatedRecurrentUnit, but it's a layer group. # it is equivalent to GatedRecurrentLayer. -def GatedRecurrentLayerGroup(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - seq_reversed = False): +def GatedRecurrentLayerGroup(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + seq_reversed=False): input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - RecurrentLayerGroupBegin(name + "_layer_group", - in_links = [input_layer_name], - out_links = [name], - seq_reversed = seq_reversed) + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + RecurrentLayerGroupBegin( + name + "_layer_group", + in_links=[input_layer_name], + out_links=[name], + seq_reversed=seq_reversed) GatedRecurrentUnit( - name = name, - size = size, - active_type = active_type, - gate_active_type = gate_active_type, - inputs = input_layer_name, #transform outside - para_prefix = para_prefix, - error_clipping_threshold = error_clipping_threshold, - ) + name=name, + size=size, + active_type=active_type, + gate_active_type=gate_active_type, + inputs=input_layer_name, #transform outside + para_prefix=para_prefix, + error_clipping_threshold=error_clipping_threshold, ) RecurrentLayerGroupEnd(name + "_layer_group") - diff --git a/python/paddle/trainer_config_helpers/__init__.py b/python/paddle/trainer_config_helpers/__init__.py index 451b9ac3396eadf9fab2b5fd940a6f924e042976..adebebba2523f851507c4a0525eeaae9cfeb9dcc 100644 --- a/python/paddle/trainer_config_helpers/__init__.py +++ b/python/paddle/trainer_config_helpers/__init__.py @@ -20,3 +20,6 @@ from layers import * from networks import * from optimizers import * from attrs import * + +# This will enable operator overload for LayerOutput +import math diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index ad5cdc0a0eb13f7a58e7d89ebfb79d33a63b75d5..6261934e1bc8e8df62aeaa0757f4a237f91ef748 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -12,20 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ["TanhActivation", "SigmoidActivation", - "SoftmaxActivation", "IdentityActivation", "LinearActivation", - 'SequenceSoftmaxActivation', 'ExpActivation', - "ReluActivation", "BReluActivation", "SoftReluActivation", - "STanhActivation", - "AbsActivation", "SquareActivation", - "BaseActivation"] +__all__ = [ + "TanhActivation", "SigmoidActivation", "SoftmaxActivation", + "IdentityActivation", "LinearActivation", 'SequenceSoftmaxActivation', + 'ExpActivation', "ReluActivation", "BReluActivation", "SoftReluActivation", + "STanhActivation", "AbsActivation", "SquareActivation", "BaseActivation" +] class BaseActivation(object): """ - A mark for activation class. + A mark for activation class. Each activation inherit BaseActivation, which has two parameters. - + :param name: activation name in paddle config. :type name: basestring :param support_hppl: True if supported by hppl. HPPL is a library used by paddle @@ -51,7 +50,8 @@ class TanhActivation(BaseActivation): f(z)=tanh(z)=\\frac{e^z-e^{-z}}{e^z+e^{-z}} """ - def __init__(self): BaseActivation.__init__(self, 'tanh', True) + def __init__(self): + BaseActivation.__init__(self, 'tanh', True) class SigmoidActivation(BaseActivation): @@ -63,7 +63,8 @@ class SigmoidActivation(BaseActivation): f(z) = \\frac{1}{1+exp(-z)} """ - def __init__(self): BaseActivation.__init__(self, 'sigmoid', True) + def __init__(self): + BaseActivation.__init__(self, 'sigmoid', True) class SoftmaxActivation(BaseActivation): @@ -104,7 +105,8 @@ class IdentityActivation(BaseActivation): Just do nothing for output both forward/backward. """ - def __init__(self): BaseActivation.__init__(self, '', False) + def __init__(self): + BaseActivation.__init__(self, '', False) LinearActivation = IdentityActivation @@ -124,7 +126,8 @@ class ReluActivation(BaseActivation): 0 &\\quad\\mathrm{otherwize} """ - def __init__(self): BaseActivation.__init__(self, 'relu', True) + def __init__(self): + BaseActivation.__init__(self, 'relu', True) class BReluActivation(BaseActivation): @@ -141,7 +144,8 @@ class BReluActivation(BaseActivation): 0 &\\quad \\mathrm{otherwise} """ - def __init__(self): BaseActivation.__init__(self, 'brelu', False) + def __init__(self): + BaseActivation.__init__(self, 'brelu', False) class SoftReluActivation(BaseActivation): @@ -149,7 +153,9 @@ class SoftReluActivation(BaseActivation): SoftRelu Activation. """ - def __init__(self): BaseActivation.__init__(self, 'softrelu', False) + def __init__(self): + BaseActivation.__init__(self, 'softrelu', False) + class STanhActivation(BaseActivation): """ @@ -160,7 +166,8 @@ class STanhActivation(BaseActivation): f(z) = 1.7159 * tanh(2/3*z) """ - def __init__(self): BaseActivation.__init__(self, 'stanh', False) + def __init__(self): + BaseActivation.__init__(self, 'stanh', False) class AbsActivation(BaseActivation): @@ -178,7 +185,8 @@ class AbsActivation(BaseActivation): 0 &\\quad if \\quad z = 0 """ - def __init__(self): BaseActivation.__init__(self, 'abs', False) + def __init__(self): + BaseActivation.__init__(self, 'abs', False) class SquareActivation(BaseActivation): @@ -189,16 +197,21 @@ class SquareActivation(BaseActivation): f(z) = z^2. """ - def __init__(self): BaseActivation.__init__(self, 'square', False) + def __init__(self): + BaseActivation.__init__(self, 'square', False) + class ExpActivation(BaseActivation): """ Exponential Activation. - + .. math:: f(z) = e^z. """ - def __init__(self): BaseActivation.__init__(self, 'exponential', False) + + def __init__(self): + BaseActivation.__init__(self, 'exponential', False) + class LogActivation(BaseActivation): """ @@ -207,4 +220,6 @@ class LogActivation(BaseActivation): .. math:: f(z) = log(z) """ - def __init__(self): BaseActivation.__init__(self, 'log', False) + + def __init__(self): + BaseActivation.__init__(self, 'log', False) diff --git a/python/paddle/trainer_config_helpers/attrs.py b/python/paddle/trainer_config_helpers/attrs.py index d26344124733246c67790025fb186c6b350c3947..54169f382f164e7b9cf061baeb21d4109a8ae5b6 100644 --- a/python/paddle/trainer_config_helpers/attrs.py +++ b/python/paddle/trainer_config_helpers/attrs.py @@ -13,8 +13,9 @@ # limitations under the License. from paddle.trainer.config_parser import * -__all__ = ['ParamAttr', 'ExtraAttr', 'ParameterAttribute', - 'ExtraLayerAttribute'] +__all__ = [ + 'ParamAttr', 'ExtraAttr', 'ParameterAttribute', 'ExtraLayerAttribute' +] def convert_and_compare(x, Type): @@ -25,7 +26,8 @@ def convert_and_compare(x, Type): :param Type: target type to check x over """ - return type(x)(Type(x))==x + return type(x)(Type(x)) == x + def is_compatible_with(x, Type): """ @@ -38,9 +40,9 @@ def is_compatible_with(x, Type): return True try: if float == Type or int == Type: - # avoid those types that can be converted to float/int but not very - # meaningful and could potentially lead to error - # i.e., str and bool typed value should not be used for initializing float/int variable + # avoid those types that can be converted to float/int but not very + # meaningful and could potentially lead to error + # i.e., str and bool typed value should not be used for initializing float/int variable if not isinstance(x, str) and not isinstance(x, bool): return convert_and_compare(x, Type) elif bool == Type: @@ -91,9 +93,17 @@ class ParameterAttribute(object): :type sparse_update: bool """ - def __init__(self, name=None, is_static=False, initial_std=None, - initial_mean=None, initial_max=None, initial_min=None, - l1_rate=None, l2_rate=None, learning_rate=None, momentum=None, + def __init__(self, + name=None, + is_static=False, + initial_std=None, + initial_mean=None, + initial_max=None, + initial_min=None, + l1_rate=None, + l2_rate=None, + learning_rate=None, + momentum=None, sparse_update=False): # initialize strategy. if is_static: @@ -183,7 +193,10 @@ class ExtraLayerAttribute(object): :type device: int """ - def __init__(self, error_clipping_threshold=None, drop_rate=None, device=None): + def __init__(self, + error_clipping_threshold=None, + drop_rate=None, + device=None): self.attr = dict() if isinstance(error_clipping_threshold, float): assert error_clipping_threshold > 0 @@ -200,8 +213,8 @@ class ExtraLayerAttribute(object): for key in self.attr: if not hasattr(self, 'can_%s' % key) or \ not getattr(self, 'can_%s' % key): - raise NotImplementedError( - "Layer %s cannot support %s" % (layer_name, key)) + raise NotImplementedError("Layer %s cannot support %s" % + (layer_name, key)) @staticmethod def to_kwargs(attr): diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index f51140656d0dcfed14c67fd6f1d60351ba5e8ab2..b41097953dad8aa9c8755c25860b177cdbff5b93 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Data Sources are helpers to define paddle training data or testing data. """ @@ -26,8 +25,12 @@ except ImportError: __all__ = ['define_py_data_sources2'] -def define_py_data_source(file_list, cls, module, - obj, args=None, async=False, +def define_py_data_source(file_list, + cls, + module, + obj, + args=None, + async=False, data_cls=PyData): """ Define a python data source. @@ -76,8 +79,9 @@ def define_py_data_source(file_list, cls, module, args = pickle.dumps(args, 0) if data_cls is None: + def py_data2(files, load_data_module, load_data_object, load_data_args, - **kwargs): + **kwargs): data = DataBase() data.type = 'py2' data.files = files @@ -86,17 +90,25 @@ def define_py_data_source(file_list, cls, module, data.load_data_args = load_data_args data.async_load_data = True return data - data_cls = py_data2 - - cls(data_cls(files=file_list, - load_data_module=module, - load_data_object=obj, - load_data_args=args, - async_load_data=async)) + data_cls = py_data2 -def define_py_data_sources(train_list, test_list, module, obj, args=None, - train_async=False, data_cls=PyData): + cls( + data_cls( + files=file_list, + load_data_module=module, + load_data_object=obj, + load_data_args=args, + async_load_data=async)) + + +def define_py_data_sources(train_list, + test_list, + module, + obj, + args=None, + train_async=False, + data_cls=PyData): """ The annotation is almost the same as define_py_data_sources2, except that it can specific train_async and data_cls. @@ -125,8 +137,8 @@ def define_py_data_sources(train_list, test_list, module, obj, args=None, """ def __is_splitable__(o): - return (isinstance(o, list) or isinstance(o, tuple) - ) and hasattr(o, '__len__') and len(o) == 2 + return (isinstance(o, list) or + isinstance(o, tuple)) and hasattr(o, '__len__') and len(o) == 2 assert train_list is not None or test_list is not None assert module is not None and obj is not None @@ -139,7 +151,7 @@ def define_py_data_sources(train_list, test_list, module, obj, args=None, test_obj = obj train_obj = obj if __is_splitable__(obj): - train_module, test_module = module + train_obj, test_obj = obj if args is None: args = "" @@ -196,9 +208,10 @@ def define_py_data_sources2(train_list, test_list, module, obj, args=None): :return: None :rtype: None """ - define_py_data_sources(train_list=train_list, - test_list=test_list, - module=module, - obj=obj, - args=args, - data_cls=None) + define_py_data_sources( + train_list=train_list, + test_list=test_list, + module=module, + obj=obj, + args=args, + data_cls=None) diff --git a/python/paddle/trainer_config_helpers/default_decorators.py b/python/paddle/trainer_config_helpers/default_decorators.py index be00f48b457c137e3b0913da84ad2e6215f9e9ca..c01050e338d5933f49f0504f2e9ef5f15c7743ba 100644 --- a/python/paddle/trainer_config_helpers/default_decorators.py +++ b/python/paddle/trainer_config_helpers/default_decorators.py @@ -18,16 +18,18 @@ from .attrs import ParamAttr from .activations import TanhActivation from paddle.trainer.config_parser import * -__all__ = ['wrap_name_default', 'wrap_param_attr_default', - 'wrap_bias_attr_default', 'wrap_act_default', - 'wrap_param_default'] +__all__ = [ + 'wrap_name_default', 'wrap_param_attr_default', 'wrap_bias_attr_default', + 'wrap_act_default', 'wrap_param_default' +] def __default_not_set_callback__(kwargs, name): return name not in kwargs or kwargs[name] is None -def wrap_param_default(param_names=None, default_factory=None, +def wrap_param_default(param_names=None, + default_factory=None, not_set_callback=__default_not_set_callback__): assert param_names is not None assert isinstance(param_names, list) or isinstance(param_names, tuple) @@ -43,7 +45,8 @@ def wrap_param_default(param_names=None, default_factory=None, if argspec.defaults: num_positional -= len(argspec.defaults) if not argspec.varargs and len(args) > num_positional: - logger.fatal("Must use keyword arguments for non-positional args") + logger.fatal( + "Must use keyword arguments for non-positional args") for name in param_names: if not_set_callback(kwargs, name): # Not set kwargs[name] = default_factory(func) @@ -112,13 +115,13 @@ def wrap_param_attr_default(param_names=None, default_factory=None): return wrap_param_default(param_names, default_factory) -def wrap_bias_attr_default(param_names=None, default_factory=None, +def wrap_bias_attr_default(param_names=None, + default_factory=None, has_bias=True): if param_names is None: param_names = ['bias_attr'] if default_factory is None: - default_factory = lambda _: ParamAttr(initial_std=0., - initial_mean=0.) + default_factory = lambda _: ParamAttr(initial_std=0., initial_mean=0.) def __bias_attr_not_set__(kwargs, name): if has_bias: diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index ded124a5c8ca44af02cd7df81ee2bff87af98337..dc6a36392f9c6bff42d3a37f963ed18a849414f5 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -15,13 +15,14 @@ from paddle.trainer.config_parser import * from default_decorators import * -__all__ = ["evaluator_base","classification_error_evaluator", "auc_evaluator", - "pnpair_evaluator", "precision_recall_evaluator", - "ctc_error_evaluator", "chunk_evaluator", "sum_evaluator", - "column_sum_evaluator", "value_printer_evaluator", - "gradient_printer_evaluator", "maxid_printer_evaluator", - "maxframe_printer_evaluator", "seqtext_printer_evaluator", - "classification_error_printer_evaluator"] +__all__ = [ + "evaluator_base", "classification_error_evaluator", "auc_evaluator", + "pnpair_evaluator", "precision_recall_evaluator", "ctc_error_evaluator", + "chunk_evaluator", "sum_evaluator", "column_sum_evaluator", + "value_printer_evaluator", "gradient_printer_evaluator", + "maxid_printer_evaluator", "maxframe_printer_evaluator", + "seqtext_printer_evaluator", "classification_error_printer_evaluator" +] class EvaluatorAttribute(object): @@ -32,10 +33,7 @@ class EvaluatorAttribute(object): FOR_UTILS = 1 << 4 KEYS = [ - "for_classification", - "for_regression", - "for_rank", - "for_print", + "for_classification", "for_regression", "for_rank", "for_print", "for_utils" ] @@ -55,22 +53,23 @@ def evaluator(*attrs): setattr(method, EvaluatorAttribute.to_key(attr), True) method.is_evaluator = True return method + return impl -def evaluator_base( - input, - type, - label=None, - weight=None, - name=None, - chunk_scheme=None, - num_chunk_types=None, - classification_threshold=None, - positive_label=None, - dict_file=None, - result_file=None, - num_results=None, - delimited=None): + +def evaluator_base(input, + type, + label=None, + weight=None, + name=None, + chunk_scheme=None, + num_chunk_types=None, + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None): """ Evaluator will evaluate the network status while training/testing. @@ -130,14 +129,14 @@ def evaluator_base( result_file=result_file, delimited=delimited) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() -def classification_error_evaluator( - input, - label, - name=None, - weight=None, - threshold=None): +def classification_error_evaluator(input, + label, + name=None, + weight=None, + threshold=None): """ Classification Error Evaluator. It will print error rate for classification. @@ -170,13 +169,14 @@ def classification_error_evaluator( :return: None. """ - evaluator_base(name=name, - type="classification_error", - input=input, - label=label, - weight=weight, - classification_threshold=threshold, - ) + evaluator_base( + name=name, + type="classification_error", + input=input, + label=label, + weight=weight, + classification_threshold=threshold, ) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -184,8 +184,7 @@ def auc_evaluator( input, label, name=None, - weight=None, - ): + weight=None, ): """ Auc Evaluator which adapts to binary classification. @@ -205,11 +204,13 @@ def auc_evaluator( [sample_num, 1]. :type weight: LayerOutput """ - evaluator_base(name=name, - type="last-column-auc", - input=input, - label=label, - weight=weight) + evaluator_base( + name=name, + type="last-column-auc", + input=input, + label=label, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_RANK) @wrap_name_default() @@ -218,8 +219,7 @@ def pnpair_evaluator( label, info, name=None, - weight=None, - ): + weight=None, ): """ Positive-negative pair rate Evaluator which adapts to rank task like learning to rank. This evaluator must contain at least three layers. @@ -242,12 +242,14 @@ def pnpair_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="pnpair", - input=input, - label=label, - info=info, - weight=weight) + evaluator_base( + name=name, + type="pnpair", + input=input, + label=label, + info=info, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -256,8 +258,7 @@ def precision_recall_evaluator( label, positive_label=None, weight=None, - name=None, - ): + name=None, ): """ An Evaluator to calculate precision and recall, F1-score. It is adapt to the task with multiple labels. @@ -286,20 +287,21 @@ def precision_recall_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="precision_recall", - input=input, - label=label, - positive_label=positive_label, - weight=weight) + evaluator_base( + name=name, + type="precision_recall", + input=input, + label=label, + positive_label=positive_label, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() def ctc_error_evaluator( input, label, - name=None, - ): + name=None, ): """ This evaluator is to calculate sequence-to-sequence edit distance. @@ -317,10 +319,9 @@ def ctc_error_evaluator( label for ctc_layer :type label: LayerOutput """ - evaluator_base(name=name, - type="ctc_edit_distance", - input=input, - label=label) + evaluator_base( + name=name, type="ctc_edit_distance", input=input, label=label) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -328,8 +329,7 @@ def chunk_evaluator( input, name=None, chunk_scheme=None, - num_chunk_types=None, - ): + num_chunk_types=None, ): """ Chunk evaluator is used to evaluate segment labelling accuracy for a sequence. It calculates the chunk detection F1 score. @@ -375,19 +375,20 @@ def chunk_evaluator( :type chunk_scheme: basestring :param num_chunk_types: number of chunk types other than "other" """ - evaluator_base(name=name, - type="chunk", - input=input, - chunk_scheme=chunk_scheme, - num_chunk_types=num_chunk_types) + evaluator_base( + name=name, + type="chunk", + input=input, + chunk_scheme=chunk_scheme, + num_chunk_types=num_chunk_types) + @evaluator(EvaluatorAttribute.FOR_UTILS) @wrap_name_default() def sum_evaluator( input, name=None, - weight=None, - ): + weight=None, ): """ An Evaluator to sum the result of input. @@ -405,18 +406,15 @@ def sum_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="sum", - input=input, - weight=weight) + evaluator_base(name=name, type="sum", input=input, weight=weight) + @evaluator(EvaluatorAttribute.FOR_UTILS) @wrap_name_default() def column_sum_evaluator( input, name=None, - weight=None, - ): + weight=None, ): """ This Evaluator is used to sum the last column of input. @@ -431,22 +429,22 @@ def column_sum_evaluator( :param input: Input Layer name. :type input: LayerOutput """ - evaluator_base(name=name, - type="last-column-sum", - input=input, - weight=weight) + evaluator_base( + name=name, type="last-column-sum", input=input, weight=weight) + """ The following are printer Evaluators which are usually used to print the result, like value or gradient of input layers, the results generated in machine translation, the classification error etc. """ + + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def value_printer_evaluator( input, - name=None, - ): + name=None, ): """ This Evaluator is used to print the values of input layers. It contains one or more input layers. @@ -462,16 +460,14 @@ def value_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="value_printer", - input=input) + evaluator_base(name=name, type="value_printer", input=input) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def gradient_printer_evaluator( input, - name=None, - ): + name=None, ): """ This Evaluator is used to print the gradient of input layers. It contains one or more input layers. @@ -487,17 +483,15 @@ def gradient_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="gradient_printer", - input=input) + evaluator_base(name=name, type="gradient_printer", input=input) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxid_printer_evaluator( input, num_results=None, - name=None, - ): + name=None, ): """ This Evaluator is used to print maximum top k values and their indexes of each row of input layers. It contains one or more input layers. @@ -517,18 +511,16 @@ def maxid_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="max_id_printer", - input=input, - num_results=num_results) + evaluator_base( + name=name, type="max_id_printer", input=input, num_results=num_results) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxframe_printer_evaluator( input, num_results=None, - name=None, - ): + name=None, ): """ This Evaluator is used to print the top k frames of each input layers. The input layers should contain sequences info or sequences type. @@ -549,10 +541,12 @@ def maxframe_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="max_frame_printer", - input=input, - num_results=num_results) + evaluator_base( + name=name, + type="max_frame_printer", + input=input, + num_results=num_results) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() @@ -562,8 +556,7 @@ def seqtext_printer_evaluator( id_input=None, dict_file=None, delimited=None, - name=None, - ): + name=None, ): """ Sequence text printer will print text according to index matrix and a dictionary. There can be multiple input to this layer: @@ -636,12 +629,14 @@ def seqtext_printer_evaluator( inputs = [id_input, input] input.parents.append(id_input) - evaluator_base(name=name, - type="seq_text_printer", - input=inputs, - dict_file=dict_file, - result_file=result_file, - delimited=delimited) + evaluator_base( + name=name, + type="seq_text_printer", + input=inputs, + dict_file=dict_file, + result_file=result_file, + delimited=delimited) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() @@ -649,8 +644,7 @@ def classification_error_printer_evaluator( input, label, threshold=0.5, - name=None, - ): + name=None, ): """ This Evaluator is used to print the classification error of each sample. @@ -667,8 +661,9 @@ def classification_error_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="classification_error_printer", - input=input, - label=label, - classification_threshold=threshold) + evaluator_base( + name=name, + type="classification_error_printer", + input=input, + label=label, + classification_threshold=threshold) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index ccfdb3ded3ba1a0e2bf980d7dd5aed5619b68089..d984e843204c1cd99ee5b8941dc056c091504869 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -29,34 +29,84 @@ except ImportError: import pickle import copy -__all__ = ["full_matrix_projection", "AggregateLevel", "ExpandLevel", - "identity_projection", "dotmul_projection", "dotmul_operator", - "table_projection", "mixed_layer", "data_layer", - "embedding_layer", "fc_layer", "grumemory", - "pooling_layer", "lstmemory", "last_seq", "first_seq", - "cos_sim", "hsigmoid", "conv_projection", - "regression_cost", 'classification_cost', "LayerOutput", - 'img_conv_layer', 'img_pool_layer', 'batch_norm_layer', - 'img_cmrnorm_layer', 'addto_layer', - 'concat_layer', 'lstm_step_layer', 'recurrent_group', - 'memory', 'StaticInput', 'expand_layer', 'scaling_layer', - 'power_layer', 'interpolation_layer', 'bilinear_interp_layer', - 'trans_layer', 'sum_to_one_norm_layer', - 'get_output_layer', 'LayerType', 'context_projection', - 'beam_search', 'maxid_layer', 'GeneratedInput', 'SubsequenceInput', - 'gru_step_layer', 'recurrent_layer', - 'BaseGeneratedInput', 'conv_operator', 'conv_shift_layer', - 'tensor_layer', 'selective_fc_layer', 'sampling_id_layer', - 'slope_intercept_layer', 'trans_full_matrix_projection', - 'linear_comb_layer', - 'convex_comb_layer', 'ctc_layer', 'crf_layer', 'crf_decoding_layer', - 'nce_layer', - 'cross_entropy_with_selfnorm', 'cross_entropy', - 'multi_binary_label_cross_entropy', - 'rank_cost', 'lambda_cost', 'huber_cost', - 'block_expand_layer', - 'maxout_layer', 'out_prod_layer', 'print_layer' - ] +__all__ = [ + "full_matrix_projection", + "AggregateLevel", + "ExpandLevel", + "identity_projection", + "dotmul_projection", + "dotmul_operator", + "repeat_layer", + "table_projection", + "mixed_layer", + "data_layer", + "embedding_layer", + "fc_layer", + "grumemory", + "pooling_layer", + "lstmemory", + "last_seq", + "first_seq", + "cos_sim", + "hsigmoid", + "conv_projection", + "regression_cost", + 'classification_cost', + "LayerOutput", + 'img_conv_layer', + 'img_pool_layer', + 'batch_norm_layer', + 'img_cmrnorm_layer', + 'addto_layer', + 'concat_layer', + 'lstm_step_layer', + 'recurrent_group', + 'memory', + 'StaticInput', + 'expand_layer', + 'scaling_layer', + 'scaling_projection', + 'power_layer', + 'interpolation_layer', + 'bilinear_interp_layer', + 'trans_layer', + 'sum_to_one_norm_layer', + 'get_output_layer', + 'LayerType', + 'context_projection', + 'beam_search', + 'maxid_layer', + 'GeneratedInput', + 'SubsequenceInput', + 'gru_step_layer', + 'recurrent_layer', + 'BaseGeneratedInput', + 'conv_operator', + 'conv_shift_layer', + 'tensor_layer', + 'selective_fc_layer', + 'sampling_id_layer', + 'slope_intercept_layer', + 'trans_full_matrix_projection', + 'linear_comb_layer', + 'convex_comb_layer', + 'ctc_layer', + 'crf_layer', + 'crf_decoding_layer', + 'nce_layer', + 'cross_entropy_with_selfnorm', + 'cross_entropy', + 'multi_binary_label_cross_entropy', + 'sum_cost', + 'rank_cost', + 'lambda_cost', + 'huber_cost', + 'block_expand_layer', + 'maxout_layer', + 'out_prod_layer', + 'print_layer', + 'spp_layer', +] class LayerType(object): @@ -78,6 +128,7 @@ class LayerType(object): COSINE_SIM = 'cos' HSIGMOID = 'hsigmoid' CONV_LAYER = "conv" + CONVTRANS_LAYER = "convt" POOL_LAYER = "pool" BATCH_NORM_LAYER = 'batch_norm' NORM_LAYER = 'norm' @@ -98,6 +149,7 @@ class LayerType(object): SCALING_LAYER = 'scaling' TRANS_LAYER = 'trans' OUT_PROD_LAYER = 'out_prod' + FEATURE_MAP_EXPAND_LAYER = 'featmap_expand' MEMORY = 'memory' MAXID_LAYER = 'maxid' @@ -112,6 +164,7 @@ class LayerType(object): LINEAR_COMBINATION_LAYER = "convex_comb" BLOCK_EXPAND = "blockexpand" MAXOUT = "maxout" + SPP_LAYER = "spp" PRINT_LAYER = "print" @@ -127,6 +180,7 @@ class LayerType(object): CROSS_ENTROPY_WITH_SELFNORM = "multi_class_cross_entropy_with_selfnorm" SOFT_BIN_CLASS_CROSS_ENTROPY = "soft_binary_class_cross_entropy" MULTI_BIN_LABEL_CROSS_ENTROPY = "multi_binary_label_cross_entropy" + SUM_COST = "sum_cost" @staticmethod def is_layer_type(type_name): @@ -175,11 +229,19 @@ class LayerOutput(object): :type parents: list|tuple|collections.Sequence """ - def __init__(self, name, layer_type, parents=None, activation=None, - num_filters=None, img_norm_type=None, size=None, outputs=None, + def __init__(self, + name, + layer_type, + parents=None, + activation=None, + num_filters=None, + img_norm_type=None, + size=None, + outputs=None, reverse=None): assert isinstance(name, basestring) assert isinstance(layer_type, basestring) + assert size is not None assert LayerType.is_layer_type(layer_type) self.name = name self.layer_type = layer_type @@ -216,6 +278,7 @@ DEVICE = 'device' def layer_support(*attrs): attrs_list = list(attrs) attrs_list.append(DEVICE) + def decorator(method): @functools.wraps(method) def wrapper(*args, **kwargs): @@ -275,9 +338,8 @@ def full_matrix_projection(input, size=0, param_attr=None): :return: A FullMatrixProjection Object. :rtype: FullMatrixProjection """ - proj = FullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = FullMatrixProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -312,9 +374,8 @@ def trans_full_matrix_projection(input, size=0, param_attr=None): :return: A TransposedFullMatrixProjection Object. :rtype: TransposedFullMatrixProjection """ - proj = TransposedFullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = TransposedFullMatrixProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -358,9 +419,8 @@ def table_projection(input, size=0, param_attr=None): :return: A TableProjection Object. :rtype: TableProjection """ - proj = TableProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = TableProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -399,19 +459,46 @@ def identity_projection(input, offset=None): :type input: LayerOutput :param offset: Offset, None if use default. :type offset: int - :return: A IdentityProjection or IdentityOffsetProjection Object + :return: A IdentityProjection or IdentityOffsetProjection object :rtype: IdentityProjection or IdentityOffsetProjection """ if offset is None: proj = IdentityProjection(input_layer_name=input.name) proj.origin = input else: - proj = IdentityOffsetProjection(input_layer_name=input.name, - offset=offset) + proj = IdentityOffsetProjection( + input_layer_name=input.name, offset=offset) proj.origin = input return proj +@wrap_param_attr_default() +def scaling_projection(input, param_attr=None): + """ + scaling_projection multiplies the input with a scalar parameter and add to + the output. + + .. math:: + out += w * in + + The example usage is: + + .. code-block:: python + + proj = scaling_projection(input=layer) + + :param input: Input Layer. + :type input: LayerOutput + :param param_attr: Parameter config, None if use default. + :type param_attr: ParameterAttribute + :return: A ScalingProjection object + :rtype: ScalingProjection + """ + proj = ScalingProjection(input_layer_name=input.name, **param_attr.attr) + proj.origin = input + return proj + + @wrap_param_attr_default() def dotmul_projection(input, param_attr=None): """ @@ -436,9 +523,8 @@ def dotmul_projection(input, param_attr=None): :return: A DotMulProjection Object. :rtype: DotMulProjection """ - proj = DotMulProjection(input_layer_name=input.name, - size=input.size, - **param_attr.attr) + proj = DotMulProjection( + input_layer_name=input.name, size=input.size, **param_attr.attr) proj.origin = input return proj @@ -471,21 +557,22 @@ def dotmul_operator(a=None, b=None, scale=1, **kwargs): if 'x' in kwargs or 'y' in kwargs: logger.warning('x and y arguments for dotmul_operator is deprecated. ' 'Please use a and b as parameter.') - a = kwargs.get('x', a) # For Backward capacity. + a = kwargs.get('x', a) # For Backward capacity. b = kwargs.get('y', b) assert isinstance(a, LayerOutput) assert isinstance(b, LayerOutput) if a.size is not None and b.size is not None: assert a.size == b.size - op = DotMulOperator(input_layer_names=[a.name, b.name], - scale=scale) + op = DotMulOperator(input_layer_names=[a.name, b.name], scale=scale) op.origin = [a, b] return op @wrap_bias_attr_default(['padding_attr']) -def context_projection(input, context_len, context_start=None, +def context_projection(input, + context_len, + context_start=None, padding_attr=False): """ Context Projection. @@ -522,11 +609,12 @@ def context_projection(input, context_len, context_start=None, if trainable: extra_dict = padding_attr.attr - proj = ContextProjection(input_layer_name=input.name, - context_length=context_len, - context_start=context_start, - trainable_padding=trainable, - **extra_dict) + proj = ContextProjection( + input_layer_name=input.name, + context_length=context_len, + context_start=context_start, + trainable_padding=trainable, + **extra_dict) proj.origin = input return proj @@ -540,8 +628,7 @@ class MixedLayerType(LayerOutput): def __init__(self): Exception.__init__(self) - def __init__(self, name, size, act, bias_attr, layer_attr, - parents=None): + def __init__(self, name, size, act, bias_attr, layer_attr, parents=None): """ Ctor. :param name: layer name. @@ -558,8 +645,13 @@ class MixedLayerType(LayerOutput): :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute or None """ - LayerOutput.__init__(self, name, LayerType.MIXED_LAYER, parents, - size=size, activation=act) + LayerOutput.__init__( + self, + name, + LayerType.MIXED_LAYER, + parents, + size=size, + activation=act) self.bias_attr = bias_attr self.layer_attr = layer_attr self.inputs = [] @@ -591,21 +683,27 @@ class MixedLayerType(LayerOutput): def __exit__(self, *args, **kwargs): del args, kwargs # unused parameter to suppress warning assert len(self.inputs) != 0 - MixedLayer( + ml = MixedLayer( name=self.name, size=self.size, active_type=self.activation.name, bias=ParamAttr.to_bias(self.bias_attr), inputs=self.inputs, - **ExtraLayerAttribute.to_kwargs(self.layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(self.layer_attr)) + # update the size which might be computed inside MixedLayer + # according to the operator's output size + self.size = ml.config.size @wrap_name_default("mixed") @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(ERROR_CLIPPING, DROPOUT) -def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, +def mixed_layer(size=0, + input=None, + name=None, + act=None, + bias_attr=False, layer_attr=None): """ Mixed Layer. A mixed layer will add all inputs together, then activate. @@ -650,8 +748,12 @@ def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, if input is None: return MixedLayerType(name, size, act, bias_attr, layer_attr) else: - with mixed_layer(name=name, size=size, act=act, bias_attr=bias_attr, - layer_attr=layer_attr) as m: + with mixed_layer( + name=name, + size=size, + act=act, + bias_attr=bias_attr, + layer_attr=layer_attr) as m: if isinstance(input, collections.Sequence): for each in input: m += each @@ -681,8 +783,11 @@ def data_layer(name, size, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer(type=LayerType.DATA, name=name, size=size, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + type=LayerType.DATA, + name=name, + size=size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.DATA, size=size) @@ -708,9 +813,12 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - with mixed_layer(name=name, size=size, act=LinearActivation(), - bias_attr=False, - layer_attr=layer_attr) as mix: + with mixed_layer( + name=name, + size=size, + act=LinearActivation(), + bias_attr=False, + layer_attr=layer_attr) as mix: mix += table_projection(input=input, size=size, param_attr=param_attr) return mix @@ -720,8 +828,13 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): @wrap_bias_attr_default() @wrap_act_default() @layer_support(ERROR_CLIPPING, DROPOUT) -def fc_layer(input, size, act=None, name=None, - param_attr=None, bias_attr=None, layer_attr=None): +def fc_layer(input, + size, + act=None, + name=None, + param_attr=None, + bias_attr=None, + layer_attr=None): """ Helper for declare fully connected layer. @@ -773,17 +886,17 @@ def fc_layer(input, size, act=None, name=None, assert isinstance(input, collections.Sequence) Layer( - inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( - input, param_attr)], + inputs=[ + Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr) + ], name=name, type=LayerType.FC_LAYER, size=size, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.FC_LAYER, input, activation=act, - size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.FC_LAYER, input, activation=act, size=size) @wrap_name_default("print") @@ -806,8 +919,7 @@ def print_layer(input, name=None): Layer( name=name, type=LayerType.PRINT_LAYER, - inputs=[l.name for l in input], - ) + inputs=[l.name for l in input], ) # this layer don't return anything, can not be input of other layer. @@ -815,7 +927,10 @@ def print_layer(input, name=None): @wrap_bias_attr_default(has_bias=False) @wrap_param_default(['pooling_type'], default_factory=lambda _: MaxPooling()) @layer_support() -def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, +def pooling_layer(input, + pooling_type=None, + name=None, + bias_attr=None, agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ @@ -862,23 +977,27 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, inputs=[Input(input.name)], bias=ParamAttr.to_bias(bias_attr), trans_type=agg_level, - **extra_dict - ) + **extra_dict) - return LayerOutput(name, pooling_type.name, parents=[input], - size=input.size) + return LayerOutput( + name, pooling_type.name, parents=[input], size=input.size) @wrap_bias_attr_default() @wrap_param_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(param_names=["act", 'state_act'], act=TanhActivation()) @wrap_name_default("lstmemory") @layer_support(DROPOUT) -def lstmemory(input, name=None, reverse=False, act=None, - gate_act=None, size=None, - state_act=None, bias_attr=None, param_attr=None, +def lstmemory(input, + name=None, + reverse=False, + act=None, + gate_act=None, + size=None, + state_act=None, + bias_attr=None, + param_attr=None, layer_attr=None): """ Long Short-term Memory Cell. @@ -953,30 +1072,38 @@ def lstmemory(input, name=None, reverse=False, act=None, "layer. The lstm size should be equal with input layer size/4. The" " size which is set explicitly will be ignored." % name) - Layer(name=name, - type=LayerType.LSTMEMORY, - active_type=act.name, - active_state_type=state_act.name, - active_gate_type=gate_act.name, - reversed=reverse, - bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input.name, **param_attr.attr)], - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type=LayerType.LSTMEMORY, + active_type=act.name, + active_state_type=state_act.name, + active_gate_type=gate_act.name, + reversed=reverse, + bias=ParamAttr.to_bias(bias_attr), + inputs=[Input(input.name, **param_attr.attr)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.LSTMEMORY, [input], size=input.size / 4, - reverse=reverse) + return LayerOutput( + name, + LayerType.LSTMEMORY, [input], + size=input.size / 4, + reverse=reverse) @wrap_bias_attr_default() @wrap_param_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(param_names=["act"], act=TanhActivation()) @wrap_name_default("gru") @layer_support(DROPOUT) -def grumemory(input, name=None, reverse=False, act=None, - gate_act=None, size=None, - bias_attr=None, param_attr=None, +def grumemory(input, + name=None, + reverse=False, + act=None, + gate_act=None, + size=None, + bias_attr=None, + param_attr=None, layer_attr=None): """ Gate Recurrent Unit Layer. @@ -1067,23 +1194,28 @@ def grumemory(input, name=None, reverse=False, act=None, " and should be input size / 3. Set size explicitly will be " "ignored.") - Layer(name=name, - type=LayerType.GRUMEMORY, - active_type=act.name, - active_gate_type=gate_act.name, - reversed=reverse, - bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input.name, **param_attr.attr)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + Layer( + name=name, + type=LayerType.GRUMEMORY, + active_type=act.name, + active_gate_type=gate_act.name, + reversed=reverse, + bias=ParamAttr.to_bias(bias_attr), + inputs=[Input(input.name, **param_attr.attr)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.GRUMEMORY, [input], size=input.size / 3, - reverse=reverse) + return LayerOutput( + name, + LayerType.GRUMEMORY, [input], + size=input.size / 3, + reverse=reverse) @wrap_name_default() @layer_support() -def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, +def last_seq(input, + name=None, + agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ Get Last Timestamp Activation of a sequence. @@ -1109,15 +1241,19 @@ def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, type=LayerType.SEQUENCE_LAST_INSTANCE, inputs=[input.name], trans_type=agg_level, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEQUENCE_LAST_INSTANCE, parents=[input], - size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEQUENCE_LAST_INSTANCE, + parents=[input], + size=input.size) @wrap_name_default() @layer_support() -def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, +def first_seq(input, + name=None, + agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ Get First Timestamp Activation of a sequence. @@ -1144,10 +1280,12 @@ def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, type=LayerType.SEQUENCE_FIRST_INSTANCE, inputs=[input.name], trans_type=agg_level, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEQUENCE_FIRST_INSTANCE, - parents=[input], size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEQUENCE_FIRST_INSTANCE, + parents=[input], + size=input.size) class ExpandLevel(object): @@ -1157,7 +1295,8 @@ class ExpandLevel(object): @wrap_name_default() @layer_support() -def expand_layer(input, expand_as, +def expand_layer(input, + expand_as, name=None, bias_attr=False, expand_level=ExpandLevel.FROM_TIMESTEP, @@ -1197,12 +1336,53 @@ def expand_layer(input, expand_as, bias=ParamAttr.to_bias(bias_attr=bias_attr), type=LayerType.EXPAND_LAYER, trans_type=expand_level, - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name=name, - size=input.size, - layer_type=LayerType.EXPAND_LAYER, - parents=[input, expand_as]) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + size=input.size, + layer_type=LayerType.EXPAND_LAYER, + parents=[input, expand_as]) + + +@wrap_name_default() +@layer_support() +def repeat_layer(input, num_repeats, name=None, layer_attr=None): + """ + A layer for repeating the input for num_repeats times. This is equivalent + to apply concat_layer() with num_repeats same input. + + .. math:: + y = [x, x, \cdots, x] + + The example usage is: + + .. code-block:: python + + expand = repeat_layer(layer, 4) + + :param input: Input layer + :type input: LayerOutput + :param num_repeats: Repeat the input so many times + :type num_repeats: int + :param name: Layer name. + :type name: basestring + :param layer_attr: extra layer attributes. + :type layer_attr: ExtraLayerAttribute. + :return: LayerOutput object. + :rtype: LayerOutput + """ + + l = Layer( + inputs=[input.name], + name=name, + num_filters=num_repeats, + type=LayerType.FEATURE_MAP_EXPAND_LAYER, + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + size=l.config.size, + layer_type=LayerType.FEATURE_MAP_EXPAND_LAYER, + parents=[input]) @wrap_name_default() @@ -1249,11 +1429,12 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.INTERPOLATION_LAYER, inputs=[weight.name, input[0].name, input[1].name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.INTERPOLATION_LAYER, - parents=[weight, input[0], input[1]], - size=input[0].size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.INTERPOLATION_LAYER, + parents=[weight, input[0], input[1]], + size=input[0].size) @wrap_name_default() @@ -1273,11 +1454,11 @@ def bilinear_interp_layer(input, .. code-block:: python bilinear = bilinear_interp_layer(input=layer1, out_size_x=64, out_size_y=64) - + :param input: A input layer. :type input: LayerOutput. :param out_size_x: bilinear interpolation output width. - :type out_size_x: int|None + :type out_size_x: int|None :param out_size_y: bilinear interpolation output height. :type out_size_y: int|None :param name: The layer's name, which cna not be specified. @@ -1292,15 +1473,23 @@ def bilinear_interp_layer(input, assert out_size_x > 0 and out_size_y > 0 assert input.num_filters is not None num_channels = input.num_filters - Layer(name=name, - inputs=Input(input.name, - bilinear_interp=BilinearInterp(out_size_x=out_size_x, - out_size_y=out_size_y, - num_channels=num_channels)), - type=LayerType.BILINEAR_INTERP_LAYER, - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input], - num_filters=num_channels) + l = Layer( + name=name, + inputs=Input( + input.name, + bilinear_interp=BilinearInterp( + out_size_x=out_size_x, + out_size_y=out_size_y, + num_channels=num_channels)), + type=LayerType.BILINEAR_INTERP_LAYER, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.BILINEAR_INTERP_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + @wrap_name_default() @layer_support() @@ -1339,10 +1528,9 @@ def power_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.POWER_LAYER, inputs=[weight.name, input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.POWER_LAYER, - parents=[input, weight], size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.POWER_LAYER, parents=[input, weight], size=input.size) @wrap_name_default() @@ -1384,10 +1572,9 @@ def scaling_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.SCALING_LAYER, inputs=[weight.name, input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SCALING_LAYER, parents=[weight, input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SCALING_LAYER, parents=[weight, input], size=input.size) @wrap_name_default() @@ -1420,10 +1607,9 @@ def trans_layer(input, name=None, layer_attr=None): name=name, type=LayerType.TRANS_LAYER, inputs=[input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.TRANS_LAYER, parents=[input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.TRANS_LAYER, parents=[input], size=input.size) @wrap_name_default() @@ -1465,8 +1651,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): type=LayerType.COSINE_SIM, cos_scale=scale, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) else: if a.size is not None and b.size is not None: assert size == b.size / a.size @@ -1476,17 +1661,21 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): size=size, cos_scale=scale, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b]) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b], size=size) @wrap_name_default() @wrap_bias_attr_default(has_bias=True) @wrap_param_attr_default() @layer_support() -def hsigmoid(input, label, num_classes, name=None, bias_attr=None, - param_attr=None, layer_attr=None): +def hsigmoid(input, + label, + num_classes, + name=None, + bias_attr=None, + param_attr=None, + layer_attr=None): """ Organize the classes into a binary tree. At each node, a sigmoid function is used to calculate the probability of belonging to the right branch. @@ -1541,15 +1730,15 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, ipts_for_layer.append(label.name) parents.append(label) - Layer( + l = Layer( name=name, type=LayerType.HSIGMOID, num_classes=num_classes, bias=ParamAttr.to_bias(bias_attr), inputs=ipts_for_layer, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.HSIGMOID, parents=parents) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.HSIGMOID, parents=parents, size=l.config.size) @wrap_name_default("conv") @@ -1557,11 +1746,23 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, @wrap_bias_attr_default() @wrap_act_default(act=ReluActivation()) @layer_support(DROPOUT) -def img_conv_layer(input, filter_size, num_filters, - name=None, num_channels=None, - act=None, groups=1, stride=1, padding=0, bias_attr=None, - param_attr=None, shared_biases=True, layer_attr=None, - filter_size_y=None, stride_y=None, padding_y=None): +def img_conv_layer(input, + filter_size, + num_filters, + name=None, + num_channels=None, + act=None, + groups=1, + stride=1, + padding=0, + bias_attr=None, + param_attr=None, + shared_biases=True, + layer_attr=None, + filter_size_y=None, + stride_y=None, + padding_y=None, + trans=False): """ Convolution layer for image. Paddle only support square input currently and thus input image's width equals height. @@ -1570,6 +1771,13 @@ def img_conv_layer(input, filter_size, num_filters, `_ . + Convolution Transpose (deconv) layer for image. Paddle only support square + input currently and thus input image's width equals height. + + The details of convolution transpose layer, + please refer to the following explanation and references therein + `_ . The num_channel means input image's channel number. It may be 1 or 3 when input is raw pixels of image(mono or RGB), or it may be the previous layer's num_filters * num_group. @@ -1619,6 +1827,8 @@ def img_conv_layer(input, filter_size, num_filters, :type shared_biases: bool :param layer_attr: Layer Extra Attribute. :type layer_attr: ExtraLayerAttribute + :param trans: true if it is a convTransLayer, false if it is a convLayer + :type trans: bool :return: LayerOutput object. :rtype: LayerOutput """ @@ -1649,36 +1859,56 @@ def img_conv_layer(input, filter_size, num_filters, if param_attr.attr.get('initial_smart'): # special initial for conv layers. - init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + init_w = (2.0 / (filter_size**2 * num_channels))**0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False - Layer( + + lt = LayerType.CONVTRANS_LAYER if trans else LayerType.CONV_LAYER + + l = Layer( name=name, - inputs=Input(input.name, conv=Conv( - filter_size=filter_size, padding=padding, stride=stride, - channels=num_channels, groups=groups, - filter_size_y=filter_size_y, padding_y=padding_y, - stride_y=stride_y), - **param_attr.attr), + inputs=Input( + input.name, + conv=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + groups=groups, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y), + **param_attr.attr), active_type=act.name, num_filters=num_filters, bias=ParamAttr.to_bias(bias_attr), shared_biases=shared_biases, - type=LayerType.CONV_LAYER, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.CONV_LAYER, parents=[input], - activation=act, num_filters=num_filters) + type=lt, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + lt, + parents=[input], + activation=act, + num_filters=num_filters, + size=l.config.size) @wrap_name_default("pool") @layer_support() -def img_pool_layer(input, pool_size, name=None, - num_channels=None, pool_type=None, - stride=1, padding=0, layer_attr=None, - pool_size_y=None, stride_y=None, padding_y=None, +def img_pool_layer(input, + pool_size, + name=None, + num_channels=None, + pool_type=None, + stride=1, + padding=0, + layer_attr=None, + pool_size_y=None, + stride_y=None, + padding_y=None, img_width=None): """ Image pooling Layer. @@ -1701,7 +1931,7 @@ def img_pool_layer(input, pool_size, name=None, :type pool_size_y: int|None :param num_channels: number of input channel. :type num_channels: int - :param pool_type: pooling type. MaxPooling or AveragePooling. Default is + :param pool_type: pooling type. MaxPooling or AvgPooling. Default is MaxPooling. :type pool_type: BasePoolingType :param stride: stride width of pooling. @@ -1733,51 +1963,133 @@ def img_pool_layer(input, pool_size, name=None, stride_y = stride if stride_y is None else stride_y padding_y = padding if padding_y is None else padding_y - Layer( + l = Layer( name=name, type=LayerType.POOL_LAYER, - inputs=[Input(input.name, - pool=Pool( - pool_type=type_name, - channels=num_channels, - size_x=pool_size, - start=None, - stride=stride, - padding=padding, - size_y=pool_size_y, - stride_y=stride_y, - padding_y=padding_y, - img_width=img_width - ))], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.POOL_LAYER, parents=[input], - num_filters=num_channels) - - -def __img_norm_layer__(name, input, size, norm_type, scale, power, - num_channels, blocked, layer_attr): + inputs=[ + Input( + input.name, + pool=Pool( + pool_type=type_name, + channels=num_channels, + size_x=pool_size, + start=None, + stride=stride, + padding=padding, + size_y=pool_size_y, + stride_y=stride_y, + padding_y=padding_y, + img_width=img_width)) + ], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.POOL_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + + +@wrap_name_default("spp") +@layer_support() +def spp_layer(input, + name=None, + num_channels=None, + pool_type=None, + pyramid_height=None, + img_width=None, + layer_attr=None): + """ + Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition. + The details please refer to + `Kaiming He's paper `_. + + :param name: layer name. + :type name: basestring + :param input: layer's input. + :type input: LayerOutput + :param num_channels: number of input channel. + :type num_channels: int + :param pool_type: Pooling type. MaxPooling or AveragePooling. Default is MaxPooling. + :type scale: BasePoolingType + :param pyramid_height: pyramid height. + :type pyramid_height: int + :param img_width: the width of input feature map. If it is None, the input feature + map should be square. + :type img_width: int|None + :param layer_attr: Extra Layer Attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters - Layer( - name=name, type=LayerType.NORM_LAYER, inputs=Input( - input.name, norm=Norm(norm_type=norm_type, - channels=num_channels, size=size, - scale=scale, - pow=power, blocked=blocked) - ), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, layer_type=LayerType.NORM_LAYER, parents=[input], - num_filters=num_channels, img_norm_type=norm_type) + if pool_type is None: + pool_type = MaxPooling() + elif isinstance(pool_type, AvgPooling): + pool_type.name = 'avg' + + type_name = pool_type.name + if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)): + type_name += '-projection' + + l = Layer( + name=name, + type=LayerType.SPP_LAYER, + inputs=Input( + input.name, + spp=SpatialPyramidPool( + pool_type=type_name, + channels=num_channels, + pyramid_height=pyramid_height, + img_width=img_width)), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.SPP_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + + +def __img_norm_layer__(name, input, size, norm_type, scale, power, num_channels, + blocked, layer_attr): + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + + l = Layer( + name=name, + type=LayerType.NORM_LAYER, + inputs=Input( + input.name, + norm=Norm( + norm_type=norm_type, + channels=num_channels, + size=size, + scale=scale, + pow=power, + blocked=blocked)), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.NORM_LAYER, + parents=[input], + num_filters=num_channels, + img_norm_type=norm_type, + size=l.config.size) @wrap_name_default("crmnorm") @layer_support() -def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, - name=None, num_channels=None, +def img_cmrnorm_layer(input, + size, + scale=0.0128, + power=0.75, + name=None, + num_channels=None, layer_attr=None): """ Response normalization across feature maps. @@ -1811,8 +2123,13 @@ def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, @wrap_act_default(act=ReluActivation()) @wrap_name_default("batch_norm") @layer_support(DROPOUT) -def batch_norm_layer(input, act=None, name=None, num_channels=None, - bias_attr=None, param_attr=None, layer_attr=None, +def batch_norm_layer(input, + act=None, + name=None, + num_channels=None, + bias_attr=None, + param_attr=None, + layer_attr=None, batch_norm_type=None, moving_average_fraction=0.9, use_global_stats=None): @@ -1896,23 +2213,25 @@ def batch_norm_layer(input, act=None, name=None, num_channels=None, num_channels = input.size assert (batch_norm_type is None) or (batch_norm_type == "batch_norm") or \ (batch_norm_type == "cudnn_batch_norm") - Layer( + l = Layer( name=name, - inputs=Input(input.name, - image=Image(channels=num_channels), - **param_attr.attr), + inputs=Input( + input.name, image=Image(channels=num_channels), **param_attr.attr), active_type=act.name, type=LayerType.BATCH_NORM_LAYER, batch_norm_type=batch_norm_type, bias=ParamAttr.to_bias(bias_attr), moving_average_fraction=moving_average_fraction, use_global_stats=use_global_stats, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.BATCH_NORM_LAYER, - parents=[input], activation=act, - num_filters=num_channels) + return LayerOutput( + name=name, + layer_type=LayerType.BATCH_NORM_LAYER, + parents=[input], + activation=act, + num_filters=num_channels, + size=l.config.size) @wrap_name_default() @@ -1947,18 +2266,16 @@ def sum_to_one_norm_layer(input, name=None, layer_attr=None): name=name, type=LayerType.SUM_TO_ONE_NORM_LAYER, inputs=[input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], size=input.size) @wrap_name_default("addto") @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(DROPOUT) -def addto_layer(input, act=None, name=None, bias_attr=None, - layer_attr=None): +def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): """ AddtoLayer. @@ -2017,15 +2334,21 @@ def addto_layer(input, act=None, name=None, bias_attr=None, if each_input.num_filters is not None: num_filters = each_input.num_filters - Layer( - name=name, type=LayerType.ADDTO_LAYER, inputs=ipts_for_layer, + l = Layer( + name=name, + type=LayerType.ADDTO_LAYER, + inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.ADDTO_LAYER, parents=input, - activation=act, num_filters=num_filters) + return LayerOutput( + name, + LayerType.ADDTO_LAYER, + parents=input, + activation=act, + num_filters=num_filters, + size=l.config.size) @wrap_act_default(act=IdentityActivation()) @@ -2084,22 +2407,22 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): LayerOutput) return a - is_concat_layer = __is_type__(reduce(__reduce_concat_type__, - map(type, input)), LayerOutput) + is_concat_layer = __is_type__( + reduce(__reduce_concat_type__, map(type, input)), LayerOutput) - layer_type = (LayerType.CONCAT_LAYER if is_concat_layer - else LayerType.CONCAT_PROJ_LAYER) + layer_type = (LayerType.CONCAT_LAYER + if is_concat_layer else LayerType.CONCAT_PROJ_LAYER) if layer_type == LayerType.CONCAT_LAYER: assert not bias_attr - + Layer( - name=name, type=layer_type, + name=name, + type=layer_type, inputs=[x.name for x in input] if is_concat_layer else input, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) sz = 0 for each_input in input: @@ -2109,14 +2432,20 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): sz = None break - return LayerOutput(name, layer_type=layer_type, - parents=input if is_concat_layer else [ - x.origin for x in input], - activation=act, size=sz) - - -def memory(name, size, is_seq=False, boot_layer=None, - boot_bias=None, boot_bias_active_type=None, + return LayerOutput( + name, + layer_type=layer_type, + parents=input if is_concat_layer else [x.origin for x in input], + activation=act, + size=sz) + + +def memory(name, + size, + is_seq=False, + boot_layer=None, + boot_bias=None, + boot_bias_active_type=None, boot_with_const_id=None): """ The memory layers is a layer cross each time step. Reference this output @@ -2164,30 +2493,33 @@ def memory(name, size, is_seq=False, boot_layer=None, assert boot_layer is None or isinstance(boot_layer, LayerOutput) - agent_name = Memory(name, size, - is_seq, - boot_layer.name if boot_layer is not None else None, - boot_bias, - boot_bias_active_type.name, - boot_with_const_id) - - lout = LayerOutput(name=agent_name, size=size, - layer_type=LayerType.MEMORY, - parents=[boot_layer] if boot_layer is not None - else None) + agent_name = Memory(name, size, is_seq, boot_layer.name + if boot_layer is not None else None, boot_bias, + boot_bias_active_type.name, boot_with_const_id) + + lout = LayerOutput( + name=agent_name, + size=size, + layer_type=LayerType.MEMORY, + parents=[boot_layer] if boot_layer is not None else None) return lout @wrap_bias_attr_default() -@wrap_act_default(param_names=['gate_act', - 'state_act'], - act=SigmoidActivation()) +@wrap_act_default( + param_names=['gate_act', 'state_act'], act=SigmoidActivation()) @wrap_act_default(act=TanhActivation()) @wrap_name_default('lstm_step') @layer_support() -def lstm_step_layer(input, state, size, act=None, - name=None, gate_act=None, state_act=None, - bias_attr=None, layer_attr=None): +def lstm_step_layer(input, + state, + size, + act=None, + name=None, + gate_act=None, + state_act=None, + bias_attr=None, + layer_attr=None): """ LSTM Step Layer. It used in recurrent_group. The lstm equations are shown as follow. @@ -2254,24 +2586,32 @@ def lstm_step_layer(input, state, size, act=None, active_gate_type=gate_act.name, active_state_type=state_act.name, bias=ParamAttr.to_bias(bias_attr), - size=size, inputs=[input.name, state.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + size=size, + inputs=[input.name, state.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.LSTM_STEP_LAYER, - parents=[input, state], activation=act, - size=size, outputs=['default', 'state']) + return LayerOutput( + name=name, + layer_type=LayerType.LSTM_STEP_LAYER, + parents=[input, state], + activation=act, + size=size, + outputs=['default', 'state']) @wrap_bias_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(act=TanhActivation()) @wrap_name_default('gru_step') @layer_support() -def gru_step_layer(input, output_mem, size=None, act=None, - name=None, gate_act=None, - bias_attr=None, layer_attr=None): +def gru_step_layer(input, + output_mem, + size=None, + act=None, + name=None, + gate_act=None, + bias_attr=None, + layer_attr=None): """ :param input: @@ -2292,20 +2632,18 @@ def gru_step_layer(input, output_mem, size=None, act=None, Layer( name=name, type=LayerType.GRU_STEP_LAYER, - inputs=[ - input.name, - output_mem.name - ], + inputs=[input.name, output_mem.name], bias=ParamAttr.to_bias(bias_attr), size=size, active_type=act.name, active_gate_type=gate_act.name, - **ExtraAttr.to_kwargs(layer_attr) - ) + **ExtraAttr.to_kwargs(layer_attr)) return LayerOutput( - name=name, layer_type=LayerType.GRU_STEP_LAYER, + name=name, + layer_type=LayerType.GRU_STEP_LAYER, parents=[input, output_mem], - size=size, activation=act) + size=size, + activation=act) @wrap_name_default() @@ -2333,13 +2671,19 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): ' The get output name is %s, which not' \ ' in %s' % ( arg_name, ",".join(input.outputs)) - Layer(name=name, type=LayerType.GET_OUTPUT_LAYER, - inputs=[Input(input.name, input_layer_argument=arg_name)], - size=input.size, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type=LayerType.GET_OUTPUT_LAYER, + inputs=[Input( + input.name, input_layer_argument=arg_name)], + size=input.size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.GET_OUTPUT_LAYER, - parents=[input], size=input.size) + return LayerOutput( + name=name, + layer_type=LayerType.GET_OUTPUT_LAYER, + parents=[input], + size=input.size) @wrap_name_default() @@ -2347,8 +2691,13 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): @wrap_bias_attr_default() @wrap_param_attr_default() @layer_support() -def recurrent_layer(input, act=None, bias_attr=None, - param_attr=None, name=None, reverse=False, layer_attr=None): +def recurrent_layer(input, + act=None, + bias_attr=None, + param_attr=None, + name=None, + reverse=False, + layer_attr=None): """ Simple recurrent unit layer. It is just a fully connect layer through both time and neural network. @@ -2383,16 +2732,21 @@ def recurrent_layer(input, act=None, bias_attr=None, :return: LayerOutput object. :rtype: LayerOutput """ - Layer(name=name, - type=LayerType.RECURRENT_LAYER, - inputs=Input(input.name, **param_attr.attr), - active_type=act.name, - bias=ParamAttr.to_bias(bias_attr), - reversed=reverse, - **ExtraAttr.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.RECURRENT_LAYER, - parents=[input], size=input.size, activation=act, - reverse=reverse) + Layer( + name=name, + type=LayerType.RECURRENT_LAYER, + inputs=Input(input.name, **param_attr.attr), + active_type=act.name, + bias=ParamAttr.to_bias(bias_attr), + reversed=reverse, + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.RECURRENT_LAYER, + parents=[input], + size=input.size, + activation=act, + reverse=reverse) class StaticInput(object): @@ -2428,7 +2782,12 @@ class SubsequenceInput(object): @wrap_name_default("recurrent_group") -def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): +def recurrent_group(step, + input, + reverse=False, + name=None, + targetInlink=None, + is_generating=False): """ Recurrent layer group is an extremely flexible recurrent unit in PaddlePaddle. As long as the user defines the calculation done within a @@ -2493,6 +2852,12 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): :type targetInlink: LayerOutput|SubsequenceInput + :param is_generating: If is generating, none of input type should be LayerOutput; + else, for training or testing, one of the input type must + be LayerOutput. + + : type is_generating: bool + :return: LayerOutput object. :rtype: LayerOutput """ @@ -2520,7 +2885,7 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): return True return False - assert(targetInlink == None or targetInlink_in_inlinks()) + assert (targetInlink == None or targetInlink_in_inlinks()) targetInlinkName = None if targetInlink == None \ else targetInlink.name if isinstance(targetInlink, LayerOutput) \ else targetInlink.input.name @@ -2535,10 +2900,12 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): return x.name RecurrentLayerGroupWithoutOutLinksBegin( - name=name, in_links=map(map_in_links, in_links), + name=name, + in_links=map(map_in_links, in_links), seq_reversed=reverse, target_inlinkname=targetInlinkName) in_args = [] + has_LayerOutput = True for each_input in input: assert is_single_input(each_input) if isinstance(each_input, LayerOutput): @@ -2546,16 +2913,22 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): elif isinstance(each_input, SubsequenceInput): in_args.append(each_input.input) else: + has_LayerOutput = False mem_name = "__%s_memory__" % each_input.input.name - mem = memory(name=mem_name, - is_seq=each_input.is_seq, - size=each_input.input.size, - boot_layer=each_input.input) - with mixed_layer(name=mem_name, size=each_input.input.size, - act=IdentityActivation()) as mix: + mem = memory( + name=mem_name, + is_seq=each_input.is_seq, + size=each_input.input.size, + boot_layer=each_input.input) + with mixed_layer( + name=mem_name, + size=each_input.input.size, + act=IdentityActivation()) as mix: mix += identity_projection(mem) in_args.append(mem) + assert (is_generating != has_LayerOutput) + layer_outs = step(*in_args) if isinstance(layer_outs, LayerOutput): @@ -2594,14 +2967,15 @@ class GeneratedInput(BaseGeneratedInput): return maxid_layer(input=input, name='__beam_search_predict__') def before_real_step(self): - predict_id = memory(name='__beam_search_predict__', - size=self.size, - boot_with_const_id=self.bos_id) - - trg_emb = embedding_layer(input=predict_id, - size=self.embedding_size, - param_attr=ParamAttr( - name=self.embedding_name)) + predict_id = memory( + name='__beam_search_predict__', + size=self.size, + boot_with_const_id=self.bos_id) + + trg_emb = embedding_layer( + input=predict_id, + size=self.embedding_size, + param_attr=ParamAttr(name=self.embedding_name)) return trg_emb def __init__(self, size, embedding_name, embedding_size): @@ -2634,13 +3008,16 @@ def maxid_layer(input, name=None, layer_attr=None): """ assert isinstance(input, LayerOutput) - Layer(name=name, - type='maxid', - inputs=[input.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, - layer_type=LayerType.MAXID_LAYER, - parents=[input]) + l = Layer( + name=name, + type='maxid', + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.MAXID_LAYER, + parents=[input], + size=l.config.size) @wrap_name_default() @@ -2669,13 +3046,16 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): assert isinstance(input1, LayerOutput) assert isinstance(input2, LayerOutput) - Layer(name=name, - type="out_prod", - inputs=[input1.name, input2.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, - layer_type=LayerType.OUT_PROD_LAYER, - parents=[input1, input2]) + l = Layer( + name=name, + type=LayerType.OUT_PROD_LAYER, + inputs=[input1.name, input2.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.OUT_PROD_LAYER, + parents=[input1, input2], + size=l.config.size) @wrap_name_default() @@ -2704,18 +3084,27 @@ def eos_layer(input, eos_id, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer(name=name, - type=LayerType.EOSID_LAYER, - eos_id=eos_id, - inputs=[input.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.EOSID_LAYER, - parents=[input]) + l = Layer( + name=name, + type=LayerType.EOSID_LAYER, + eos_id=eos_id, + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.EOSID_LAYER, + parents=[input], + size=l.config.size) @wrap_name_default() -def beam_search(step, input, bos_id, eos_id, beam_size, - max_length=500, name=None, +def beam_search(step, + input, + bos_id, + eos_id, + beam_size, + max_length=500, + name=None, num_results_per_sample=None): """ Beam search is a heuristic search algorithm used in sequence generation. @@ -2789,8 +3178,7 @@ def beam_search(step, input, bos_id, eos_id, beam_size, if num_results_per_sample > beam_size: logger.warning("num_results_per_sample should be less than beam_size") - if isinstance(input, StaticInput) or isinstance(input, - BaseGeneratedInput): + if isinstance(input, StaticInput) or isinstance(input, BaseGeneratedInput): input = [input] generated_input_index = -1 @@ -2815,11 +3203,12 @@ def beam_search(step, input, bos_id, eos_id, beam_size, def __real_step__(*args): eos_name = "__%s_eos_layer__" % name - RecurrentLayerGroupSetGenerator(Generator( - eos_layer_name=eos_name, - max_num_frames=max_length, - beam_size=beam_size, - num_results_per_sample=num_results_per_sample)) + RecurrentLayerGroupSetGenerator( + Generator( + eos_layer_name=eos_name, + max_num_frames=max_length, + beam_size=beam_size, + num_results_per_sample=num_results_per_sample)) args = list(args) args.insert(generated_input_index, gipt.before_real_step()) @@ -2830,14 +3219,19 @@ def beam_search(step, input, bos_id, eos_id, beam_size, return predict - tmp = recurrent_group(step=__real_step__, input=real_input, reverse=False, - name=name) + tmp = recurrent_group( + step=__real_step__, + input=real_input, + reverse=False, + name=name, + is_generating=True) return tmp + def __cost_input__(input, label, weight=None): """ - inputs and parents for cost layers. + inputs and parents for cost layers. """ ipts = [Input(input.name), Input(label.name)] parents = [input, label] @@ -2846,12 +3240,11 @@ def __cost_input__(input, label, weight=None): ipts.append(Input(weight.name)) parents.append(weight) return ipts, parents - + @wrap_name_default() @layer_support() -def regression_cost(input, label, weight=None, name=None, - layer_attr=None): +def regression_cost(input, label, weight=None, name=None, layer_attr=None): """ Regression Layer. @@ -2873,14 +3266,20 @@ def regression_cost(input, label, weight=None, name=None, """ ipts, parents = __cost_input__(input, label, weight) - Layer(inputs=ipts, type="square_error", name=name, - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.COST, parents=parents) + Layer( + inputs=ipts, + type="square_error", + name=name, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput(name, LayerType.COST, parents=parents, size=1) @wrap_name_default("cost") @layer_support() -def classification_cost(input, label, weight=None, name=None, +def classification_cost(input, + label, + weight=None, + name=None, evaluator=classification_error_evaluator, layer_attr=None): """ @@ -2907,8 +3306,11 @@ def classification_cost(input, label, weight=None, name=None, ipts, parents = __cost_input__(input, label, weight) - Layer(name=name, type="multi-class-cross-entropy", inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type="multi-class-cross-entropy", + inputs=ipts, + **ExtraLayerAttribute.to_kwargs(layer_attr)) def __add_evaluator__(e): assert callable(e) @@ -2927,12 +3329,19 @@ def classification_cost(input, label, weight=None, name=None, for each_evaluator in evaluator: __add_evaluator__(each_evaluator) - return LayerOutput(name, LayerType.COST, parents=parents) + return LayerOutput(name, LayerType.COST, parents=parents, size=1) -def conv_operator(img, filter, filter_size, num_filters, - num_channel=None, stride=1, padding=0, - filter_size_y=None, stride_y=None, padding_y=None): +def conv_operator(img, + filter, + filter_size, + num_filters, + num_channels=None, + stride=1, + padding=0, + filter_size_y=None, + stride_y=None, + padding_y=None): """ Different from img_conv_layer, conv_op is an Operator, which can be used in mixed_layer. And conv_op takes two inputs to perform convolution. @@ -2961,8 +3370,8 @@ def conv_operator(img, filter, filter_size, num_filters, :type filter_size_y: int :param num_filters: channel of output data. :type num_filters: int - :param num_channel: channel of input data. - :type num_channel: int + :param num_channels: channel of input data. + :type num_channels: int :param stride: The x dimension of the stride. :type stride: int :param stride_y: The y dimension of the stride. @@ -2981,31 +3390,41 @@ def conv_operator(img, filter, filter_size, num_filters, if padding_y is None: padding_y = padding - if num_channel is None: - num_channel = img.num_filters + if num_channels is None: + num_channels = img.num_filters assert isinstance(filter, LayerOutput) if filter.size is not None: - filter.size = filter_size * filter_size_y * num_filters * num_channel - - op = ConvOperator(input_layer_names=[img.name, filter.name], - num_filters=num_filters, - conv_conf=Conv(filter_size=filter_size, - padding=padding, - stride=stride, - channels=num_channel, - filter_size_y=filter_size_y, - padding_y=padding_y, - stride_y=stride_y, - groups=1)) + filter.size = filter_size * filter_size_y * num_filters * num_channels + + op = ConvOperator( + input_layer_names=[img.name, filter.name], + num_filters=num_filters, + conv_conf=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=1)) op.origin = [img, filter] return op + @wrap_param_attr_default() -def conv_projection(input, filter_size, num_filters, - num_channels=None, stride=1, padding=0, - filter_size_y=None, stride_y=None, padding_y=None, - groups=1, param_attr=None): +def conv_projection(input, + filter_size, + num_filters, + num_channels=None, + stride=1, + padding=0, + filter_size_y=None, + stride_y=None, + padding_y=None, + groups=1, + param_attr=None): """ ConvProjection with a layer as input. It performs element-wise multiplication with weight. @@ -3033,8 +3452,8 @@ def conv_projection(input, filter_size, num_filters, :type filter_size_y: int :param num_filters: channel of output data. :type num_filters: int - :param num_channel: channel of input data. - :type num_channel: int + :param num_channels: channel of input data. + :type num_channels: int :param stride: The x dimension of the stride. :type stride: int :param stride_y: The y dimension of the stride. @@ -3077,23 +3496,25 @@ def conv_projection(input, filter_size, num_filters, if param_attr.attr.get('initial_smart'): # special initial for conv layers. - init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + init_w = (2.0 / (filter_size**2 * num_channels))**0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False - proj = ConvProjection(input_layer_name=input.name, - num_filters=num_filters, - conv_conf=Conv(filter_size=filter_size, - padding=padding, - stride=stride, - channels=num_channels, - filter_size_y=filter_size_y, - padding_y=padding_y, - stride_y=stride_y, - groups=groups), - **param_attr.attr) + proj = ConvProjection( + input_layer_name=input.name, + num_filters=num_filters, + conv_conf=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=groups), + **param_attr.attr) proj.origin = input return proj @@ -3141,11 +3562,10 @@ def conv_shift_layer(a, b, name=None, layer_attr=None): name=name, type=LayerType.CONV_SHIFT_LAYER, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], - size=a.size) + return LayerOutput( + name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], size=a.size) @wrap_name_default() @@ -3153,8 +3573,14 @@ def conv_shift_layer(a, b, name=None, layer_attr=None): @wrap_bias_attr_default() @wrap_act_default(act=LinearActivation()) @layer_support(ERROR_CLIPPING, DROPOUT) -def tensor_layer(a, b, size, act=None, name=None, - param_attr=None, bias_attr=None, layer_attr=None): +def tensor_layer(a, + b, + size, + act=None, + name=None, + param_attr=None, + bias_attr=None, + layer_attr=None): """ This layer performs tensor operation for two input. For example, each sample: @@ -3203,12 +3629,10 @@ def tensor_layer(a, b, size, act=None, name=None, type=LayerType.TENSOR_LAYER, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(a.name, **param_attr.attr), - Input(b.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.TENSOR_LAYER, parents=[a, b], - activation=act, size=size) + inputs=[Input(a.name, **param_attr.attr), Input(b.name)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.TENSOR_LAYER, parents=[a, b], activation=act, size=size) @wrap_name_default() @@ -3216,11 +3640,17 @@ def tensor_layer(a, b, size, act=None, name=None, @wrap_bias_attr_default() @wrap_act_default() @layer_support() -def selective_fc_layer(input, select, size, act=None, name=None, +def selective_fc_layer(input, + select, + size, + act=None, + name=None, pass_generation=False, has_selected_colums=True, mul_ratio=0.02, - param_attr=None, bias_attr=None, layer_attr=None): + param_attr=None, + bias_attr=None, + layer_attr=None): """ Selectived fully connected layer. Different from fc_layer, the output of this layer maybe sparse. It requires an additional input to indicate @@ -3270,8 +3700,9 @@ def selective_fc_layer(input, select, size, act=None, name=None, if select.size is not None: assert select.size == size Layer( - inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( - input, param_attr)] + [select.name], + inputs=[ + Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr) + ] + [select.name], name=name, type=LayerType.SEL_FC_LAYER, size=size, @@ -3280,11 +3711,13 @@ def selective_fc_layer(input, select, size, act=None, name=None, selective_fc_pass_generation=pass_generation, has_selected_colums=has_selected_colums, selective_fc_full_mul_ratio=mul_ratio, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEL_FC_LAYER, list(input) + [select], - activation=act, - size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEL_FC_LAYER, + list(input) + [select], + activation=act, + size=size) @wrap_name_default() @@ -3309,18 +3742,21 @@ def sampling_id_layer(input, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer( + l = Layer( name=name, type=LayerType.SAMPLING_ID_LAYER, inputs=[Input(input.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SAMPLING_ID_LAYER, input) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SAMPLING_ID_LAYER, input, size=l.config.size) @wrap_name_default() @layer_support() -def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0, +def slope_intercept_layer(input, + name=None, + slope=1.0, + intercept=0.0, layer_attr=None): """ This layer for applying a slope and an intercept to the input @@ -3354,15 +3790,14 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0, slope=slope, intercept=intercept, inputs=[Input(input.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SLOPE_INTERCEPT_LAYER, input) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SLOPE_INTERCEPT_LAYER, input, size=input.size) @wrap_name_default() @layer_support() -def linear_comb_layer(weights, vectors, size=None, name=None, - layer_attr=None): +def linear_comb_layer(weights, vectors, size=None, name=None, layer_attr=None): """ A layer for weighted sum of vectors takes two inputs. - Input: size of weights is M @@ -3372,6 +3807,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None, .. math:: z(i) = \sum_{j=0}^{M-1} x(j) y(i+Nj) + where :math:`0 \le i \le N-1` Or in the matrix notation: @@ -3412,7 +3848,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None, if vectors.size is not None and weights.size is not None: assert vectors.size % weights.size == 0 if size is None: - size = vectors.size / weights.size + size = vectors.size / weights.size else: assert size == vectors.size / weights.size Layer( @@ -3420,10 +3856,9 @@ def linear_comb_layer(weights, vectors, size=None, name=None, type=LayerType.LINEAR_COMBINATION_LAYER, size=size, inputs=[Input(weights.name), Input(vectors.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.LINEAR_COMBINATION_LAYER, - [weights, vectors], size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.LINEAR_COMBINATION_LAYER, [weights, vectors], size=size) convex_comb_layer = linear_comb_layer @@ -3495,20 +3930,23 @@ def block_expand_layer(input, if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters - Layer(name=name, - inputs=Input(input.name, - block_expand=BlockExpand(channels=num_channels, - block_x=block_x, - block_y=block_y, - stride_x=stride_x, - stride_y=stride_y, - padding_x=padding_x, - padding_y=padding_y)), - type=LayerType.BLOCK_EXPAND, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - - return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input]) + l = Layer( + name=name, + inputs=Input( + input.name, + block_expand=BlockExpand( + channels=num_channels, + block_x=block_x, + block_y=block_y, + stride_x=stride_x, + stride_y=stride_y, + padding_x=padding_x, + padding_y=padding_y)), + type=LayerType.BLOCK_EXPAND, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + + return LayerOutput( + name, LayerType.BLOCK_EXPAND, parents=[input], size=l.config.size) @wrap_name_default() @@ -3525,15 +3963,15 @@ def maxout_layer(input, - Input: output of a conv layer. - Output: feature map size same as input. Channel is (input channel) / groups. - So groups should be larger than 1, and the num of channels should be able + So groups should be larger than 1, and the num of channels should be able to devided by groups. - Please refer to Paper: + Please refer to Paper: - Maxout Networks: http://www.jmlr.org/proceedings/papers/v28/goodfellow13.pdf - Multi-digit Number Recognition from Street View \ Imagery using Deep Convolutional Neural Networks: \ https://arxiv.org/pdf/1312.6082v4.pdf - + The simple usage is: .. code-block:: python @@ -3569,18 +4007,24 @@ def maxout_layer(input, assert input.num_filters is not None num_channels = input.num_filters assert num_channels % groups == 0 - Layer(name=name, - inputs=Input(input.name, - maxout=MaxOut(channels=num_channels, - groups=groups)), - type=LayerType.MAXOUT, - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.MAXOUT, parents=[input]) + l = Layer( + name=name, + inputs=Input( + input.name, maxout=MaxOut( + channels=num_channels, groups=groups)), + type=LayerType.MAXOUT, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.MAXOUT, parents=[input], size=l.config.size) @wrap_name_default() @layer_support() -def ctc_layer(input, label, size=None, name=None, norm_by_times=False, +def ctc_layer(input, + label, + size=None, + name=None, + norm_by_times=False, layer_attr=None): """ Connectionist Temporal Classification (CTC) is designed for temporal @@ -3636,15 +4080,19 @@ def ctc_layer(input, label, size=None, name=None, norm_by_times=False, size=size, norm_by_times=norm_by_times, inputs=[input.name, label.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size) @wrap_name_default() @wrap_param_attr_default() @layer_support() -def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, +def crf_layer(input, + label, + size=None, + weight=None, + param_attr=None, + name=None, layer_attr=None): """ A layer for calculating the cost of sequential conditional random @@ -3686,8 +4134,7 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, else: assert size == input.size - ipts = [Input(input.name, **param_attr.attr), - Input(label.name)] + ipts = [Input(input.name, **param_attr.attr), Input(label.name)] if weight is not None: ipts.append(Input(weight.name)) @@ -3696,18 +4143,24 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, type=LayerType.CRF_LAYER, size=size, inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input, label] if weight is not None: parents.append(weight) - return LayerOutput(name, LayerType.CRF_LAYER, parents, size=size) + # The size for LayerOutput means the dimension of the output. + # It's different from the meaning of crf layer, which is the number of + # classes. + return LayerOutput(name, LayerType.CRF_LAYER, parents, size=1) @wrap_name_default() @wrap_param_attr_default() @layer_support() -def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, +def crf_decoding_layer(input, + size, + label=None, + param_attr=None, + name=None, layer_attr=None): """ A layer for calculating the decoding sequence of sequential conditional @@ -3744,19 +4197,28 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, type=LayerType.CRF_DECODING_LAYER, size=size, inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input] if label is not None: parents.append(label) - return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=size) + # The size for LayerOutput means the dimension of the output. + # It's different from the meaning of crf layer, which is the number of + # classes. + return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=1) + @wrap_bias_attr_default(has_bias=True) @wrap_name_default() @layer_support() -def nce_layer(input, label, num_classes, weight=None, - num_neg_samples=10, neg_distribution=None, - name=None, bias_attr=None, layer_attr=None): +def nce_layer(input, + label, + num_classes, + weight=None, + num_neg_samples=10, + neg_distribution=None, + name=None, + bias_attr=None, + layer_attr=None): """ Noise-contrastive estimation. Implements the method in the following paper: @@ -3778,9 +4240,9 @@ def nce_layer(input, label, num_classes, weight=None, :param weight: weight layer, can be None(default) :type weight: LayerOutput :param num_classes: number of classes. - :type num_classes: int + :type num_classes: int :param num_neg_samples: number of negative samples. Default is 10. - :type num_neg_samples: int + :type num_neg_samples: int :param neg_distribution: The distribution for generating the random negative labels. A uniform distribution will be used if not provided. If not None, its length must be equal to num_classes. @@ -3801,7 +4263,7 @@ def nce_layer(input, label, num_classes, weight=None, assert isinstance(neg_distribution, collections.Sequence) assert len(neg_distribution) == num_classes assert sum(neg_distribution) == 1 - + ipts_for_layer = [] parents = [] for each_input in input: @@ -3817,7 +4279,7 @@ def nce_layer(input, label, num_classes, weight=None, ipts_for_layer.append(weight.name) parents.append(weight) - Layer( + l = Layer( name=name, type=LayerType.NCE_LAYER, num_classes=num_classes, @@ -3825,9 +4287,10 @@ def nce_layer(input, label, num_classes, weight=None, num_neg_samples=num_neg_samples, inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.NCE_LAYER, parents=parents) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.NCE_LAYER, parents=parents, size=l.config.size) + """ following are cost Layers. @@ -3836,7 +4299,13 @@ following are cost Layers. @wrap_name_default() @layer_support() -def rank_cost(left, right, label, weight=None, name=None, coeff=1.0, layer_attr=None): +def rank_cost(left, + right, + label, + weight=None, + name=None, + coeff=1.0, + layer_attr=None): """ A cost Layer for learning to rank using gradient descent. Details can refer to `papers $protostr/$conf.protostr.unitest done + +for conf in ${whole_configs[*]} +do + echo "Generating " $conf + python -m paddle.utils.dump_config $conf.py "" --whole > $protostr/$conf.protostr.unitest +done diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_layers.py b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py index f33357c3906fdbbebb1b4995e84115ff4edef581..9fda16a5407a1fe0af8c5986023a8368e5b87222 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/img_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py @@ -1,22 +1,23 @@ from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-3, - batch_size=1000 -) +settings(learning_rate=1e-3, batch_size=1000) -img = data_layer(name='image', size=256*256) +img = data_layer(name='image', size=256 * 256) # the parse_conv in config_parse.py is not strictly accurate when filter_size # is not square. So here set square filter_size. -img_conv = img_conv_layer(input=img, num_channels=1, num_filters=64, - filter_size=(32, 32), padding=(1, 1), stride=(1, 1), - act=LinearActivation()) +img_conv = img_conv_layer( + input=img, + num_channels=1, + num_filters=64, + filter_size=(32, 32), + padding=(1, 1), + stride=(1, 1), + act=LinearActivation()) img_bn = batch_norm_layer(input=img_conv, act=ReluActivation()) img_norm = img_cmrnorm_layer(input=img_bn, size=32) img_pool = img_pool_layer(input=img_conv, pool_size=32, pool_type=MaxPooling()) - outputs(img_pool, img_norm) diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py b/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..91849b40a0801b07642f96c061755597cd2ec073 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py @@ -0,0 +1,24 @@ +from paddle.trainer_config_helpers import * + +settings(learning_rate=1e-3, batch_size=1000) + +img = data_layer(name='image', size=227 * 227) + +# the parse_conv in config_parse.py is not strictly accurate when filter_size +# is not square. So here set square filter_size. +img_conv = img_conv_layer( + input=img, + num_channels=1, + num_filters=64, + filter_size=(32, 32), + padding=(1, 1), + stride=(1, 1), + act=LinearActivation(), + trans=True) +img_bn = batch_norm_layer(input=img_conv, act=ReluActivation()) + +img_norm = img_cmrnorm_layer(input=img_bn, size=32) + +img_pool = img_pool_layer(input=img_conv, pool_size=32, pool_type=MaxPooling()) + +outputs(img_pool, img_norm) diff --git a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py index d54a1c49fd3fdf9eb8a675dd94561e6da5b310bc..3a1a0132b64bb928e857905f99c0be2e81ccbda2 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py +++ b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py @@ -1,21 +1,12 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-5 -) +settings(batch_size=1000, learning_rate=1e-5) din = data_layer(name='data', size=30) -seq_op = [ - first_seq, - last_seq -] +seq_op = [first_seq, last_seq] -agg_level = [ - AggregateLevel.EACH_SEQUENCE, - AggregateLevel.EACH_TIMESTEP -] +agg_level = [AggregateLevel.EACH_SEQUENCE, AggregateLevel.EACH_TIMESTEP] opts = [] @@ -23,4 +14,4 @@ for op in seq_op: for al in agg_level: opts.append(op(input=din, agg_level=al)) -outputs(opts) \ No newline at end of file +outputs(opts) diff --git a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py index ba10dc78e1e3bf382c14c62d542256c957c1fdf5..7012dbf6a0b70957d6227d4125f4cd75b9abb215 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py +++ b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py @@ -4,18 +4,18 @@ Test all activations. from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-4, - batch_size=1000 -) +settings(learning_rate=1e-4, batch_size=1000) din = data_layer(name='input', size=100) acts = [ TanhActivation, SigmoidActivation, SoftmaxActivation, IdentityActivation, LinearActivation, ExpActivation, ReluActivation, BReluActivation, - SoftReluActivation, STanhActivation, AbsActivation, SquareActivation] + SoftReluActivation, STanhActivation, AbsActivation, SquareActivation +] -outputs( - [fc_layer(input=din, size=100, act=act(), name="layer_%d" % i) for i, act in - enumerate(acts)]) +outputs([ + fc_layer( + input=din, size=100, act=act(), name="layer_%d" % i) + for i, act in enumerate(acts) +]) diff --git a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py index fe515b7029336d093df5428ab8ac1c65a2d4e98a..c4c6d4020f59c9d39c0cfc1f075c16ac16ac33db 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py +++ b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py @@ -1,10 +1,7 @@ from paddle.trainer_config_helpers import * from paddle.trainer_config_helpers import math -settings( - batch_size=1000, - learning_rate=1e-5 -) +settings(batch_size=1000, learning_rate=1e-5) x = data_layer(name='data', size=100) x = math.exp(x) @@ -19,6 +16,11 @@ y = x + y y = y - x y = y - 2 y = 2 - y - +y = 2 * y +y = y * 3 +z = data_layer(name='data_2', size=1) +y = y * z +y = z * y +y = y + z +y = z + y outputs(y) - diff --git a/python/paddle/trainer_config_helpers/tests/configs/projections.py b/python/paddle/trainer_config_helpers/tests/configs/projections.py index 4066c5bc6e0f06e43b1c4d13020c092babdaea91..aa4521dcd5db3f845871cfaaedb02a86bcbddc38 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/projections.py +++ b/python/paddle/trainer_config_helpers/tests/configs/projections.py @@ -3,10 +3,7 @@ Test mixed layer, projections and operators. ''' from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-4 -) +settings(batch_size=1000, learning_rate=1e-4) din = data_layer(name='test', size=100) @@ -29,19 +26,22 @@ with mixed_layer() as m5: with mixed_layer() as m6: m6 += dotmul_operator(a=m3, b=m4) + m6 += scaling_projection(m3) -img = data_layer(name='img', size=32*32) -flt = data_layer(name='filter', size=3*3*1*64) +img = data_layer(name='img', size=32 * 32) +flt = data_layer(name='filter', size=3 * 3 * 1 * 64) with mixed_layer() as m7: - m7 += conv_operator(img=img, filter=flt, num_filters=64, - num_channel=1, filter_size=3) - -end = mixed_layer(input=[full_matrix_projection(input=m5), - trans_full_matrix_projection(input=m6), - full_matrix_projection(input=m7)], - size=100, - layer_attr=ExtraAttr(drop_rate=0.5, - error_clipping_threshold=40)) + m7 += conv_operator( + img=img, filter=flt, num_filters=64, num_channels=1, filter_size=3) + +end = mixed_layer( + input=[ + full_matrix_projection(input=m5), + trans_full_matrix_projection(input=m6), full_matrix_projection(input=m7) + ], + size=100, + layer_attr=ExtraAttr( + drop_rate=0.5, error_clipping_threshold=40)) outputs(end) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/img_trans_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/img_trans_layers.protostr new file mode 100644 index 0000000000000000000000000000000000000000..38346354080b02bebd937fd998fd3c63c8030346 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/img_trans_layers.protostr @@ -0,0 +1,176 @@ +type: "nn" +layers { + name: "image" + type: "data" + size: 51529 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconvt" + size: 4194304 + active_type: "" + inputs { + input_layer_name: "image" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 32 + channels: 1 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 64 + output_x: 227 + img_size: 256 + caffe_mode: true + filter_size_y: 32 + padding_y: 1 + stride_y: 1 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 64 + shared_biases: true +} +layers { + name: "__batch_norm_0__" + type: "batch_norm" + size: 4194304 + active_type: "relu" + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w0" + image_conf { + channels: 64 + img_size: 256 + } + } + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w1" + } + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w2" + } + bias_parameter_name: "___batch_norm_0__.wbias" + moving_average_fraction: 0.9 +} +layers { + name: "__crmnorm_0__" + type: "norm" + size: 4194304 + active_type: "" + inputs { + input_layer_name: "__batch_norm_0__" + norm_conf { + norm_type: "cmrnorm-projection" + channels: 64 + size: 32 + scale: 0.0004 + pow: 0.75 + output_x: 256 + img_size: 256 + blocked: false + } + } +} +layers { + name: "__pool_0__" + type: "pool" + size: 3240000 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + pool_conf { + pool_type: "max-projection" + channels: 64 + size_x: 32 + stride: 1 + output_x: 225 + img_size: 256 + padding: 0 + size_y: 32 + stride_y: 1 + output_y: 225 + img_size_y: 256 + padding_y: 0 + } + } +} +parameters { + name: "___conv_0__.w0" + size: 65536 + initial_mean: 0.0 + initial_std: 0.0441941738242 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 64 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___batch_norm_0__.w0" + size: 64 + initial_mean: 1.0 + initial_std: 0.0 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___batch_norm_0__.w1" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false + is_static: true + is_shared: true +} +parameters { + name: "___batch_norm_0__.w2" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false + is_static: true + is_shared: true +} +parameters { + name: "___batch_norm_0__.wbias" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "image" +output_layer_names: "__pool_0__" +output_layer_names: "__crmnorm_0__" +sub_models { + name: "root" + layer_names: "image" + layer_names: "__conv_0__" + layer_names: "__batch_norm_0__" + layer_names: "__crmnorm_0__" + layer_names: "__pool_0__" + input_layer_names: "image" + output_layer_names: "__pool_0__" + output_layer_names: "__crmnorm_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr index 1767445c44bf5c0ea7c1149ad9fef2dd92508c54..da8da1b541f37a09654202f68232b99e4dac9f61 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr @@ -209,8 +209,129 @@ layers { slope: 1.0 intercept: 2 } +layers { + name: "__slope_intercept_layer_6__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__slope_intercept_layer_5__" + } + slope: 2 + intercept: 0.0 +} +layers { + name: "__slope_intercept_layer_7__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__slope_intercept_layer_6__" + } + slope: 3 + intercept: 0.0 +} +layers { + name: "data_2" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__scaling_layer_0__" + type: "scaling" + size: 100 + active_type: "" + inputs { + input_layer_name: "data_2" + } + inputs { + input_layer_name: "__slope_intercept_layer_7__" + } +} +layers { + name: "__scaling_layer_1__" + type: "scaling" + size: 100 + active_type: "" + inputs { + input_layer_name: "data_2" + } + inputs { + input_layer_name: "__scaling_layer_0__" + } +} +layers { + name: "__repeat_layer_0__" + type: "featmap_expand" + size: 100 + active_type: "" + inputs { + input_layer_name: "data_2" + } + num_filters: 100 +} +layers { + name: "__mixed_2__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__scaling_layer_1__" + proj_conf { + type: "identity" + name: "___mixed_2__.w0" + input_size: 100 + output_size: 100 + } + } + inputs { + input_layer_name: "__repeat_layer_0__" + proj_conf { + type: "identity" + name: "___mixed_2__.w1" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__repeat_layer_1__" + type: "featmap_expand" + size: 100 + active_type: "" + inputs { + input_layer_name: "data_2" + } + num_filters: 100 +} +layers { + name: "__mixed_3__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_2__" + proj_conf { + type: "identity" + name: "___mixed_3__.w0" + input_size: 100 + output_size: 100 + } + } + inputs { + input_layer_name: "__repeat_layer_1__" + proj_conf { + type: "identity" + name: "___mixed_3__.w1" + input_size: 100 + output_size: 100 + } + } +} +input_layer_names: "data_2" input_layer_names: "data" -output_layer_names: "__slope_intercept_layer_5__" +output_layer_names: "__mixed_3__" sub_models { name: "root" layer_names: "data" @@ -228,8 +349,18 @@ sub_models { layer_names: "__slope_intercept_layer_3__" layer_names: "__slope_intercept_layer_4__" layer_names: "__slope_intercept_layer_5__" + layer_names: "__slope_intercept_layer_6__" + layer_names: "__slope_intercept_layer_7__" + layer_names: "data_2" + layer_names: "__scaling_layer_0__" + layer_names: "__scaling_layer_1__" + layer_names: "__repeat_layer_0__" + layer_names: "__mixed_2__" + layer_names: "__repeat_layer_1__" + layer_names: "__mixed_3__" + input_layer_names: "data_2" input_layer_names: "data" - output_layer_names: "__slope_intercept_layer_5__" + output_layer_names: "__mixed_3__" is_recurrent_layer_group: false } diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/projections.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/projections.protostr index e47e531a2223ddaa9dd1dfaf1fcee8a11008cbbd..2b3951c242411e0c0990a52bcb2ae6b1723a9367 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/projections.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/projections.protostr @@ -111,13 +111,23 @@ layers { inputs { input_layer_name: "__mixed_2__" } + inputs { + input_layer_name: "__mixed_2__" + input_parameter_name: "___mixed_5__.w1" + proj_conf { + type: "scaling" + name: "___mixed_5__.w1" + input_size: 100 + output_size: 100 + } + } inputs { input_layer_name: "__mixed_3__" } operator_confs { type: "dot_mul" input_indices: 0 - input_indices: 1 + input_indices: 2 input_sizes: 100 input_sizes: 100 output_size: 100 @@ -258,6 +268,16 @@ parameters { initial_strategy: 0 initial_smart: false } +parameters { + name: "___mixed_5__.w1" + size: 1 + initial_mean: 0.0 + initial_std: 1.0 + dims: 1 + dims: 1 + initial_strategy: 0 + initial_smart: true +} parameters { name: "___mixed_7__.w0" size: 30000 diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr index 5261cf0c44943689a957bb99c21075bb7341cd49..f6045fe1f68255daf0d9b5ab05034eec633e4503 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr @@ -23,6 +23,17 @@ layers { size: 10 active_type: "" } +layers { + name: "__fc_layer_0__" + type: "fc" + size: 4 + active_type: "tanh" + inputs { + input_layer_name: "input" + input_parameter_name: "___fc_layer_0__.w0" + } + bias_parameter_name: "___fc_layer_0__.wbias" +} layers { name: "__ctc_layer_0__" type: "ctc" @@ -36,17 +47,6 @@ layers { } norm_by_times: false } -layers { - name: "__fc_layer_0__" - type: "fc" - size: 4 - active_type: "tanh" - inputs { - input_layer_name: "input" - input_parameter_name: "___fc_layer_0__.w0" - } - bias_parameter_name: "___fc_layer_0__.wbias" -} layers { name: "crf_label" type: "data" @@ -191,6 +191,16 @@ layers { } coeff: 1.0 } +layers { + name: "__sum_cost_0__" + type: "sum_cost" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + } + coeff: 1.0 +} parameters { name: "___fc_layer_0__.w0" size: 800 @@ -241,14 +251,15 @@ output_layer_names: "__cross_entropy_0__" output_layer_names: "__cross_entropy_with_selfnorm_0__" output_layer_names: "__huber_cost_0__" output_layer_names: "__multi_binary_label_cross_entropy_0__" +output_layer_names: "__sum_cost_0__" sub_models { name: "root" layer_names: "input" layer_names: "labels" layer_names: "probs" layer_names: "xe-label" - layer_names: "__ctc_layer_0__" layer_names: "__fc_layer_0__" + layer_names: "__ctc_layer_0__" layer_names: "crf_label" layer_names: "__crf_layer_0__" layer_names: "left" @@ -264,6 +275,7 @@ sub_models { layer_names: "huber_label" layer_names: "__huber_cost_0__" layer_names: "__multi_binary_label_cross_entropy_0__" + layer_names: "__sum_cost_0__" input_layer_names: "input" input_layer_names: "labels" input_layer_names: "crf_label" @@ -284,6 +296,7 @@ sub_models { output_layer_names: "__cross_entropy_with_selfnorm_0__" output_layer_names: "__huber_cost_0__" output_layer_names: "__multi_binary_label_cross_entropy_0__" + output_layer_names: "__sum_cost_0__" is_recurrent_layer_group: false } diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_split_datasource.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_split_datasource.protostr new file mode 100644 index 0000000000000000000000000000000000000000..1cfb92255aa92fa3fbc16a816851a5c2f81c2b56 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_split_datasource.protostr @@ -0,0 +1,72 @@ +model_config { + type: "nn" + layers { + name: "a" + type: "data" + size: 10 + active_type: "" + } + input_layer_names: "a" + output_layer_names: "a" + sub_models { + name: "root" + layer_names: "a" + input_layer_names: "a" + output_layer_names: "a" + is_recurrent_layer_group: false + } +} +data_config { + type: "py2" + files: "train.list" + async_load_data: true + for_test: false + load_data_module: "a" + load_data_object: "c" + load_data_args: "" + data_ratio: 1 + is_main_data: true + usage_ratio: 1.0 +} +opt_config { + batch_size: 1000 + algorithm: "sgd" + learning_rate: 0.001 + learning_rate_decay_a: 0.0 + learning_rate_decay_b: 0.0 + l1weight: 0.1 + l2weight: 0.0 + c1: 0.0001 + backoff: 0.5 + owlqn_steps: 10 + max_backoff: 5 + l2weight_zero_iter: 0 + average_window: 0 + learning_method: "momentum" + ada_epsilon: 1e-06 + do_average_in_cpu: false + ada_rou: 0.95 + learning_rate_schedule: "poly" + delta_add_rate: 1.0 + shrink_parameter_value: 0 + adam_beta1: 0.9 + adam_beta2: 0.999 + adam_epsilon: 1e-08 + learning_rate_args: "" + async_lagged_grad_discard_ratio: 1.5 +} +test_data_config { + type: "py2" + files: "test.list" + async_load_data: true + for_test: true + load_data_module: "b" + load_data_object: "d" + load_data_args: "" + data_ratio: 1 + is_main_data: true + usage_ratio: 1.0 +} +save_dir: "./output/model" +start_pass: 0 + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_spp_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_spp_layer.protostr new file mode 100644 index 0000000000000000000000000000000000000000..8b0a8f2146b709ee67981049da8061597e1716be --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_spp_layer.protostr @@ -0,0 +1,34 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 3200 + active_type: "" +} +layers { + name: "__spp_0__" + type: "spp" + size: 80 + active_type: "" + inputs { + input_layer_name: "data" + spp_conf { + pool_type: "max-projection" + pyramid_height: 2 + channels: 16 + img_size: 10 + img_size_y: 20 + } + } +} +input_layer_names: "data" +output_layer_names: "__spp_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__spp_0__" + input_layer_names: "data" + output_layer_names: "__spp_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py b/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py index 202cf367fc7f289de5a9cd15d095d141e6010444..7c848ef3fcd63314bfe91db6ebac406ba8758998 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py +++ b/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-4, - batch_size=1000 -) +settings(learning_rate=1e-4, batch_size=1000) a = data_layer(name='feature_a', size=200) b = data_layer(name='feature_b', size=200) @@ -11,12 +8,22 @@ b = data_layer(name='feature_b', size=200) fc_param = ParamAttr(name='fc_param', initial_max=1.0, initial_min=-1.0) bias_param = ParamAttr(name='bias_param', initial_mean=0.0, initial_std=0.0) -softmax_param = ParamAttr(name='softmax_param', initial_max=1.0, initial_min=-1.0) +softmax_param = ParamAttr( + name='softmax_param', initial_max=1.0, initial_min=-1.0) -hidden_a = fc_layer(input=a, size=200, param_attr=fc_param, bias_attr=bias_param) -hidden_b = fc_layer(input=b, size=200, param_attr=fc_param, bias_attr=bias_param) +hidden_a = fc_layer( + input=a, size=200, param_attr=fc_param, bias_attr=bias_param) +hidden_b = fc_layer( + input=b, size=200, param_attr=fc_param, bias_attr=bias_param) -predict = fc_layer(input=[hidden_a, hidden_b], param_attr=[softmax_param, softmax_param], - bias_attr=False, size=10, act=SoftmaxActivation()) +predict = fc_layer( + input=[hidden_a, hidden_b], + param_attr=[softmax_param, softmax_param], + bias_attr=False, + size=10, + act=SoftmaxActivation()) -outputs(classification_cost(input=predict, label=data_layer(name='label', size=10))) +outputs( + classification_cost( + input=predict, label=data_layer( + name='label', size=10))) diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py index 8557e9daaf66ad56a51bac3d5b64ff08d861d05a..05810597b3154c3b287441465db16ee6e24b0ca2 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py +++ b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py @@ -16,14 +16,26 @@ with mixed_layer(size=400, bias_attr=False) as m2: lstm_param = ParamAttr(name='lstm_param') lstm_bias = ParamAttr(name='lstm_bias', initial_mean=0., initial_std=0.) -lstm1 = lstmemory_group(input=m1, param_attr=lstm_param, lstm_bias_attr=lstm_bias, mixed_bias_attr=False) -lstm2 = lstmemory_group(input=m2, param_attr=lstm_param, lstm_bias_attr=lstm_bias, mixed_bias_attr=False) +lstm1 = lstmemory_group( + input=m1, + param_attr=lstm_param, + lstm_bias_attr=lstm_bias, + mixed_bias_attr=False) +lstm2 = lstmemory_group( + input=m2, + param_attr=lstm_param, + lstm_bias_attr=lstm_bias, + mixed_bias_attr=False) softmax_param = ParamAttr(name='softmax_param') -predict = fc_layer(input=[last_seq(input=lstm1), last_seq(input=lstm2)], - size=10, - param_attr=[softmax_param, softmax_param], - bias_attr=False, - act=SoftmaxActivation()) -outputs(classification_cost(input=predict, label=data_layer(name='label', size=10))) +predict = fc_layer( + input=[last_seq(input=lstm1), last_seq(input=lstm2)], + size=10, + param_attr=[softmax_param, softmax_param], + bias_attr=False, + act=SoftmaxActivation()) +outputs( + classification_cost( + input=predict, label=data_layer( + name='label', size=10))) diff --git a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py index 87c2a85cf92dde19cc78d14d5d212940f11546f9..a5b5bb30b1d21aaa0c90868af7b5138e8a81aab1 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-4 -) +settings(batch_size=1000, learning_rate=1e-4) din = data_layer(name='data', size=200) @@ -13,24 +10,28 @@ rnn = recurrent_layer(input=hidden, act=SigmoidActivation()) rnn2 = recurrent_layer(input=hidden, act=SigmoidActivation(), reverse=True) -lstm1_param = fc_layer(input=hidden, size=200*4, act=LinearActivation(), - bias_attr=False) +lstm1_param = fc_layer( + input=hidden, size=200 * 4, act=LinearActivation(), bias_attr=False) lstm1 = lstmemory(input=lstm1_param, act=SigmoidActivation()) -lstm2_param = fc_layer(input=hidden, size=200*4, act=LinearActivation(), - bias_attr=False) +lstm2_param = fc_layer( + input=hidden, size=200 * 4, act=LinearActivation(), bias_attr=False) lstm2 = lstmemory(input=lstm2_param, act=SigmoidActivation(), reverse=True) -gru1_param = fc_layer(input=hidden, size=200*3, act=LinearActivation(), - bias_attr=False) +gru1_param = fc_layer( + input=hidden, size=200 * 3, act=LinearActivation(), bias_attr=False) gru1 = grumemory(input=gru1_param, act=SigmoidActivation()) -gru2_param = fc_layer(input=hidden, size=200*3, act=LinearActivation(), - bias_attr=False) +gru2_param = fc_layer( + input=hidden, size=200 * 3, act=LinearActivation(), bias_attr=False) gru2 = grumemory(input=gru2_param, act=SigmoidActivation(), reverse=True) -outputs(last_seq(input=rnn), first_seq(input=rnn2), - last_seq(input=lstm1), first_seq(input=lstm2), - last_seq(input=gru1), first_seq(gru2)) +outputs( + last_seq(input=rnn), + first_seq(input=rnn2), + last_seq(input=lstm1), + first_seq(input=lstm2), + last_seq(input=gru1), + first_seq(gru2)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py b/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py index ab9f7c4948b856d2cc7ff348fc49a9d4de3fbc3a..cd7f609638e384314177d653e46ecf7a4b41a12f 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-4 -) +settings(batch_size=1000, learning_rate=1e-4) din = data_layer(name='data', size=120) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py index 5a61c5a395af2f5fce0cda024184a7efc3171a48..e15a55b412f9459ecd89a0f654256097099c1398 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py @@ -1,30 +1,27 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-5 -) +settings(batch_size=1000, learning_rate=1e-5) data = data_layer(name='data', size=2304) -conv = img_conv_layer(input=data, - filter_size = 3, - num_channels=1, - num_filters=16, - padding=1, - act=LinearActivation(), - bias_attr=True) +conv = img_conv_layer( + input=data, + filter_size=3, + num_channels=1, + num_filters=16, + padding=1, + act=LinearActivation(), + bias_attr=True) -bilinear = bilinear_interp_layer(input=conv, - out_size_x=64, - out_size_y=64) +bilinear = bilinear_interp_layer(input=conv, out_size_x=64, out_size_y=64) -pool = img_pool_layer(input=bilinear, - num_channels=4, - pool_size=2, - stride=2, - pool_type=MaxPooling()) +pool = img_pool_layer( + input=bilinear, + num_channels=4, + pool_size=2, + stride=2, + pool_type=MaxPooling()) fc = fc_layer(input=pool, size=384, bias_attr=False) -outputs(fc) \ No newline at end of file +outputs(fc) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py index 64b45f4ded10b09ec4a7e77499e2d7b21215f430..fd979a1e9f4337417512b4d6581c34e54c3957bd 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-4, - batch_size=1000 -) +settings(learning_rate=1e-4, batch_size=1000) seq_in = data_layer(name='input', size=200) labels = data_layer(name='labels', size=5000) @@ -11,16 +8,34 @@ labels = data_layer(name='labels', size=5000) probs = data_layer(name='probs', size=10) xe_label = data_layer(name='xe-label', size=10) -outputs(ctc_layer(input=seq_in, label=labels), - crf_layer(input=fc_layer(input=seq_in, size=4), - label=data_layer(name='crf_label', size=4)), - rank_cost(left=data_layer(name='left', size=1), - right=data_layer(name='right', size=1), - label=data_layer(name='label', size=1)), - lambda_cost(input=data_layer(name='list_feature', size=100), - score=data_layer(name='list_scores', size=1)), - cross_entropy(input=probs, label=xe_label), - cross_entropy_with_selfnorm(input=probs, label=xe_label), - huber_cost(input=data_layer(name='huber_probs', size=1), - label=data_layer(name='huber_label', size=1)), - multi_binary_label_cross_entropy(input=probs, label=xe_label)) +hidden = fc_layer(input=seq_in, size=4) +outputs( + ctc_layer( + input=seq_in, label=labels), + crf_layer( + input=hidden, label=data_layer( + name='crf_label', size=4)), + rank_cost( + left=data_layer( + name='left', size=1), + right=data_layer( + name='right', size=1), + label=data_layer( + name='label', size=1)), + lambda_cost( + input=data_layer( + name='list_feature', size=100), + score=data_layer( + name='list_scores', size=1)), + cross_entropy( + input=probs, label=xe_label), + cross_entropy_with_selfnorm( + input=probs, label=xe_label), + huber_cost( + input=data_layer( + name='huber_probs', size=1), + label=data_layer( + name='huber_label', size=1)), + multi_binary_label_cross_entropy( + input=probs, label=xe_label), + sum_cost(input=hidden)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py index 29749cbb666379c2927f30cfea9ca63030159c1c..d30f70a55c5b1834074966dfb3f378e01447c8ab 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py @@ -1,14 +1,14 @@ from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-4, - batch_size=1000 -) +settings(learning_rate=1e-4, batch_size=1000) data = data_layer(name='input', size=300) lbl = data_layer(name='label', size=1) wt = data_layer(name='weight', size=1) fc = fc_layer(input=data, size=10, act=SoftmaxActivation()) -outputs(classification_cost(input=fc, label=lbl, weight=wt), - regression_cost(input=fc, label=lbl, weight=wt)) +outputs( + classification_cost( + input=fc, label=lbl, weight=wt), + regression_cost( + input=fc, label=lbl, weight=wt)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py index d9c841ab277e10fe3bbb6751d002af62862e9237..81e5161ebc165f13ebb919fd3c0fe617167be048 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py @@ -1,14 +1,12 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-5 -) +settings(batch_size=1000, learning_rate=1e-5) din = data_layer(name='data', size=30) data_seq = data_layer(name='data_seq', size=30) -outputs(expand_layer(input=din, expand_as=data_seq, - expand_level=ExpandLevel.FROM_SEQUENCE), - expand_layer(input=din, expand_as=data_seq, - expand_level=ExpandLevel.FROM_TIMESTEP)) +outputs( + expand_layer( + input=din, expand_as=data_seq, expand_level=ExpandLevel.FROM_SEQUENCE), + expand_layer( + input=din, expand_as=data_seq, expand_level=ExpandLevel.FROM_TIMESTEP)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_fc.py b/python/paddle/trainer_config_helpers/tests/configs/test_fc.py index a6d033f291d2c60086a0c6e7de2005c4acfbbc03..2842d3429c9c917845f8f4c33d3618608d40291d 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_fc.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_fc.py @@ -1,20 +1,16 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-5 -) +settings(batch_size=1000, learning_rate=1e-5) din = data_layer(name='data', size=100) trans = trans_layer(input=din) -hidden = fc_layer(input=trans, size=100, - bias_attr=False) +hidden = fc_layer(input=trans, size=100, bias_attr=False) mask = data_layer(name='mask', size=100) -hidden_sel = selective_fc_layer(input=din, select=mask, size=100, - act=SigmoidActivation()) +hidden_sel = selective_fc_layer( + input=din, select=mask, size=100, act=SigmoidActivation()) outputs(hidden, hidden_sel) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py index 8d9fd9df5179c7006afd369879301e823e11bdb5..474e4f36bad7eab13251afe265d1a7d107549efd 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py @@ -1,11 +1,13 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-4 -) +settings(batch_size=1000, learning_rate=1e-4) din = data_layer(name='data', size=120) -outputs(grumemory(input=din, size=40, reverse=True, gate_act=TanhActivation(), - act=SigmoidActivation())) +outputs( + grumemory( + input=din, + size=40, + reverse=True, + gate_act=TanhActivation(), + act=SigmoidActivation())) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py index 46069074ded56098e3fb995dda0ad360fc897900..dff1c535b3e84e14d0e7c343efe911f19872280a 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py @@ -1,11 +1,8 @@ from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-4, - batch_size=1000 -) +settings(learning_rate=1e-4, batch_size=1000) din = data_layer(name='data', size=100) label = data_layer(name='label', size=10) -outputs(hsigmoid(input=din, label=label, num_classes=10)) \ No newline at end of file +outputs(hsigmoid(input=din, label=label, num_classes=10)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py index 56304addb17b233229a0bf717378833195e3188f..7ca1cc2db365dedda5d9673cafaa851a464a7b6b 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py @@ -1,11 +1,13 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-5 -) +settings(batch_size=1000, learning_rate=1e-5) din = data_layer(name='data', size=128) -outputs(lstmemory(input=din, reverse=True, gate_act=TanhActivation(), - act=TanhActivation(), size=32)) +outputs( + lstmemory( + input=din, + reverse=True, + gate_act=TanhActivation(), + act=TanhActivation(), + size=32)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py index 7c1fb04766ae5485d246d8d7a994be3d8a6d9114..081430d716093877db6b2e44ac5417c37ede9a6e 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py @@ -1,48 +1,36 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-5 -) +settings(batch_size=1000, learning_rate=1e-5) data = data_layer(name='data', size=2304) -conv = img_conv_layer(input=data, - filter_size = 3, - num_channels=1, - num_filters=16, - padding=1, - act=LinearActivation(), - bias_attr=True) - -maxout = maxout_layer(input=conv, - num_channels=16, - groups=2) - -pool = img_pool_layer(input=maxout, - num_channels=8, - pool_size=2, - stride=2, - pool_type=MaxPooling()) - -conv2 = img_conv_layer(input=pool, - filter_size = 3, - num_channels=32, - num_filters=128, - padding=1, - act=LinearActivation(), - bias_attr=True) - -maxout2 = maxout_layer(input=conv, - num_channels=128, - groups=4) - -block = block_expand_layer(input=maxout, - num_channels=32, - stride_x=1, - stride_y=1, - block_x=1, - block_y=6) +conv = img_conv_layer( + input=data, + filter_size=3, + num_channels=1, + num_filters=16, + padding=1, + act=LinearActivation(), + bias_attr=True) + +maxout = maxout_layer(input=conv, num_channels=16, groups=2) + +pool = img_pool_layer( + input=maxout, num_channels=8, pool_size=2, stride=2, pool_type=MaxPooling()) + +conv2 = img_conv_layer( + input=pool, + filter_size=3, + num_channels=32, + num_filters=128, + padding=1, + act=LinearActivation(), + bias_attr=True) + +maxout2 = maxout_layer(input=conv, num_channels=128, groups=4) + +block = block_expand_layer( + input=maxout, num_channels=32, stride_x=1, stride_y=1, block_x=1, block_y=6) fc = fc_layer(input=block, size=384, bias_attr=False) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py index 4d8e1fdc6b598d3d0e29d3834c804e5a6976710b..b7a15666f0a5b863cbafec5f73dcfe0b9db2e0c7 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-5 -) +settings(batch_size=1000, learning_rate=1e-5) weight = data_layer(name='w', size=1) a = data_layer(name='a', size=100) @@ -11,13 +8,23 @@ b = data_layer(name='b', size=100) c = data_layer(name='c', size=200) d = data_layer(name='d', size=31) -outputs(interpolation_layer(input=[a, b], weight=weight), - power_layer(input=a, weight=weight), - scaling_layer(input=a, weight=weight), - cos_sim(a=a, b=b), - cos_sim(a=a, b=c, size=2), - sum_to_one_norm_layer(input=a), - conv_shift_layer(a=a, b=d), - tensor_layer(a=a, b=b, size=1000), - slope_intercept_layer(input=a, slope=0.7, intercept=0.9), - linear_comb_layer(weights=b, vectors=c)) +outputs( + interpolation_layer( + input=[a, b], weight=weight), + power_layer( + input=a, weight=weight), + scaling_layer( + input=a, weight=weight), + cos_sim( + a=a, b=b), + cos_sim( + a=a, b=c, size=2), + sum_to_one_norm_layer(input=a), + conv_shift_layer( + a=a, b=d), + tensor_layer( + a=a, b=b, size=1000), + slope_intercept_layer( + input=a, slope=0.7, intercept=0.9), + linear_comb_layer( + weights=b, vectors=c)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py index f6b2661c7b9e8563d3816def9490e22962e3f7cb..8da26ff44b19d0c18efae201a3b39002555d2605 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-4, - batch_size=1000 -) +settings(learning_rate=1e-4, batch_size=1000) din = data_layer(name='input', size=100) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py index 53f5c5d2499f9e6693d40b05354af83614fde8f5..60b4849d69d497109ef5af3257e212df233a2d0b 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-4, - batch_size=1000 -) +settings(learning_rate=1e-4, batch_size=1000) seq = data_layer(name='seq_input', size=100) sub_seq = data_layer(name='sub_seq_input', size=100) @@ -25,11 +22,15 @@ with mixed_layer() as lstm_param: # test lstm unit, rnn group with mixed_layer() as gru_param: gru_param += full_matrix_projection(input=seq, size=100 * 3) -outputs(last_seq(input=recurrent_group(step=generate_rnn_simple('rnn_forward'), - input=seq)), - first_seq(input=recurrent_group(step=generate_rnn_simple('rnn_back'), - input=seq, reverse=True)), - last_seq(input=recurrent_group(step=generate_rnn_simple( - 'rnn_subseq_forward'), input=SubsequenceInput(input=sub_seq))), - last_seq(input=lstmemory_group(input=lstm_param, size=100)), - last_seq(input=gru_group(input=gru_param, size=100))) +outputs( + last_seq(input=recurrent_group( + step=generate_rnn_simple('rnn_forward'), input=seq)), + first_seq(input=recurrent_group( + step=generate_rnn_simple('rnn_back'), input=seq, reverse=True)), + last_seq(input=recurrent_group( + step=generate_rnn_simple('rnn_subseq_forward'), + input=SubsequenceInput(input=sub_seq))), + last_seq(input=lstmemory_group( + input=lstm_param, size=100)), + last_seq(input=gru_group( + input=gru_param, size=100))) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py index 2e24164b5578cfe2abdb2a0ad889d5fd0c3f6e57..f67b6364d88560dfedb07428869482795be6af0c 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py @@ -1,22 +1,12 @@ from paddle.trainer_config_helpers import * -settings( - learning_rate=1e-4, - batch_size=1000 -) +settings(learning_rate=1e-4, batch_size=1000) din = data_layer(name='dat_in', size=100) -POOL_TYPE = [ - MaxPooling, - AvgPooling, - SumPooling -] +POOL_TYPE = [MaxPooling, AvgPooling, SumPooling] -AGG_LEVEL = [ - AggregateLevel.EACH_SEQUENCE, - AggregateLevel.EACH_TIMESTEP -] +AGG_LEVEL = [AggregateLevel.EACH_SEQUENCE, AggregateLevel.EACH_TIMESTEP] opts = [] @@ -24,7 +14,8 @@ for pt in POOL_TYPE: for al in AGG_LEVEL: opts.append(pooling_layer(input=din, agg_level=al, pooling_type=pt())) -opts.append(pooling_layer(input=din, - pooling_type=MaxPooling(output_max_index=True))) +opts.append( + pooling_layer( + input=din, pooling_type=MaxPooling(output_max_index=True))) outputs(opts) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py b/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py new file mode 100644 index 0000000000000000000000000000000000000000..318b4459bab7a70ddec534c4ad217161ffc72d5a --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py @@ -0,0 +1,10 @@ +from paddle.trainer_config_helpers import * + +define_py_data_sources2( + train_list="train.list", + test_list="test.list", + module=["a", "b"], + obj=("c", "d")) +settings(learning_rate=1e-3, batch_size=1000) + +outputs(data_layer(name="a", size=10)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..e20ffb584e8bdd86100455d4e55fe633b878e034 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py @@ -0,0 +1,14 @@ +from paddle.trainer_config_helpers import * + +settings(batch_size=100, learning_rate=1e-5) + +data = data_layer(name='data', size=3200) + +spp = spp_layer( + input=data, + pyramid_height=2, + num_channels=16, + pool_type=MaxPooling(), + img_width=10) + +outputs(spp) diff --git a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py index a6a3d09a4315ac3be63b8fecdb0fa359de834df0..ebb39219bdc1fa314e1d70bcda902f71296772f6 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py @@ -1,8 +1,5 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=1000, - learning_rate=1e-4 -) +settings(batch_size=1000, learning_rate=1e-4) probs = data_layer(name='probs', size=100) @@ -11,4 +8,4 @@ outputs( # It seems this layer is not correct, and should be rewrite. # block_expand_layer(input=probs, channel=1, block_x=1, block_y=3), -) \ No newline at end of file +) diff --git a/python/paddle/trainer_config_helpers/tests/configs/util_layers.py b/python/paddle/trainer_config_helpers/tests/configs/util_layers.py index aadb3f3f5e7997a56a73724b2858346e5ae17179..27f1c8e9938cdec12fccb37a3127bba1f8ee8d04 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/util_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/util_layers.py @@ -7,9 +7,7 @@ b = data_layer(name='b', size=10) result = addto_layer(input=[a, b]) concat1 = concat_layer(input=[a, b]) -concat2 = concat_layer(input=[ - identity_projection(input=a), - identity_projection(input=b) -]) +concat2 = concat_layer( + input=[identity_projection(input=a), identity_projection(input=b)]) -outputs(result, concat1, concat2) \ No newline at end of file +outputs(result, concat1, concat2) diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index faaab9107d8fbf11afafb722075acbe986efe9fd..44d134d1f7d5fb5de790cf564f4c1e0899571473 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -24,13 +24,19 @@ z = out_prod_layer(input1=x, input2=y) x1 = fc_layer(input=x, size=5) y1 = fc_layer(input=y, size=5) -z1 = mixed_layer(act=LinearActivation(), - input=[conv_operator(img=x1, - filter=y1, - filter_size=1, - num_filters=5, - num_channel=5, - stride=1)]) +z1 = mixed_layer( + act=LinearActivation(), + input=[ + conv_operator( + img=x1, + filter=y1, + filter_size=1, + num_filters=5, + num_channels=5, + stride=1) + ]) + +assert z1.size > 0 y2 = fc_layer(input=y, size=15) @@ -39,34 +45,36 @@ cos3 = cos_sim(a=x1, b=y2, size=3) linear_comb = linear_comb_layer(weights=x1, vectors=y2, size=3) -out = fc_layer(input=[cos1, cos3, linear_comb, z, z1], - size=num_classes, - act=SoftmaxActivation()) +out = fc_layer( + input=[cos1, cos3, linear_comb, z, z1], + size=num_classes, + act=SoftmaxActivation()) print_layer(input=[out]) outputs(classification_cost(out, data_layer(name="label", size=num_classes))) -dotmul = mixed_layer(input=[dotmul_operator(a=x1, b=x1), - dotmul_projection(input=y1)]) - -proj_with_attr_init = mixed_layer(input=full_matrix_projection(input=y1, - param_attr=ParamAttr(learning_rate = 0, - initial_mean = 0, - initial_std = 0)), - bias_attr = ParamAttr(initial_mean=0, initial_std=0, learning_rate=0), - act = LinearActivation(), - size = 5, - name='proj_with_attr_init') - +dotmul = mixed_layer( + input=[dotmul_operator( + a=x1, b=x1), dotmul_projection(input=y1)]) + +proj_with_attr_init = mixed_layer( + input=full_matrix_projection( + input=y1, + param_attr=ParamAttr( + learning_rate=0, initial_mean=0, initial_std=0)), + bias_attr=ParamAttr( + initial_mean=0, initial_std=0, learning_rate=0), + act=LinearActivation(), + size=5, + name='proj_with_attr_init') # for ctc -tmp = fc_layer(input=[x1, dotmul, proj_with_attr_init], - size=num_classes + 1, - act=SoftmaxActivation()) -ctc = ctc_layer(input=tmp, - label=y, - size=num_classes + 1) +tmp = fc_layer( + input=[x1, dotmul, proj_with_attr_init], + size=num_classes + 1, + act=SoftmaxActivation()) +ctc = ctc_layer(input=tmp, label=y, size=num_classes + 1) ctc_eval = ctc_error_evaluator(input=tmp, label=y) settings( @@ -74,5 +82,4 @@ settings( learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) diff --git a/python/paddle/trainer_config_helpers/utils.py b/python/paddle/trainer_config_helpers/utils.py index 40ecbb99e2b9f2a6b74ed3e94ecdfa02ada71fc3..c0235b28cdfb96fbff9c02c217ffd972e4f8816e 100644 --- a/python/paddle/trainer_config_helpers/utils.py +++ b/python/paddle/trainer_config_helpers/utils.py @@ -23,8 +23,8 @@ def deprecated(instead): @functools.wraps(func) def __wrapper__(*args, **kwargs): logger.warning("The interface %s is deprecated, " - "will be removed soon. Please use %s instead." - % (func.__name__, instead)) + "will be removed soon. Please use %s instead." % + (func.__name__, instead)) return func(*args, **kwargs) diff --git a/python/paddle/utils/__init__.py b/python/paddle/utils/__init__.py index 4f3c9434efbd80e130ecbb4d2cd3b056555b3af3..3e93f41c2e32025b3e29a0990833d7e97a7c8caa 100644 --- a/python/paddle/utils/__init__.py +++ b/python/paddle/utils/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['dump_config'] \ No newline at end of file +__all__ = ['dump_config'] diff --git a/python/paddle/utils/dump_config.py b/python/paddle/utils/dump_config.py index d8a2722575d539447fbed90d055df71863b4ba01..c5ce5c8d9a084d68b250d091808f528459f46921 100644 --- a/python/paddle/utils/dump_config.py +++ b/python/paddle/utils/dump_config.py @@ -19,13 +19,21 @@ import sys __all__ = [] if __name__ == '__main__': + whole_conf = False if len(sys.argv) == 2: conf = parse_config(sys.argv[1], '') elif len(sys.argv) == 3: conf = parse_config(sys.argv[1], sys.argv[2]) + elif len(sys.argv) == 4: + conf = parse_config(sys.argv[1], sys.argv[2]) + if sys.argv[3] == '--whole': + whole_conf = True else: raise RuntimeError() assert isinstance(conf, TrainerConfig_pb2.TrainerConfig) - print conf.model_config + if whole_conf: + print conf + else: + print conf.model_config diff --git a/python/paddle/utils/image_util.py b/python/paddle/utils/image_util.py index c545d16aafbc741bce25f9469e7f67de5b88fa8c..b5c6431c06f77cef5c31ca844a8427eebaea2fce 100644 --- a/python/paddle/utils/image_util.py +++ b/python/paddle/utils/image_util.py @@ -16,17 +16,20 @@ import numpy as np from PIL import Image from cStringIO import StringIO + def resize_image(img, target_size): """ Resize an image so that the shorter edge has length target_size. img: the input image to be resized. target_size: the target resized image size. """ - percent = (target_size/float(min(img.size[0], img.size[1]))) - resized_size = int(round(img.size[0] * percent)), int(round(img.size[1] * percent)) + percent = (target_size / float(min(img.size[0], img.size[1]))) + resized_size = int(round(img.size[0] * percent)), int( + round(img.size[1] * percent)) img = img.resize(resized_size, Image.ANTIALIAS) return img + def flip(im): """ Return the flipped image. @@ -38,6 +41,7 @@ def flip(im): else: return im[:, ::-1] + def crop_img(im, inner_size, color=True, test=True): """ Return cropped image. @@ -50,20 +54,22 @@ def crop_img(im, inner_size, color=True, test=True): If True, crop the center of images. """ if color: - height, width = max(inner_size, im.shape[1]), max(inner_size, im.shape[2]) + height, width = max(inner_size, im.shape[1]), max(inner_size, + im.shape[2]) padded_im = np.zeros((3, height, width)) startY = (height - im.shape[1]) / 2 startX = (width - im.shape[2]) / 2 endY, endX = startY + im.shape[1], startX + im.shape[2] - padded_im[:, startY: endY, startX: endX] = im + padded_im[:, startY:endY, startX:endX] = im else: im = im.astype('float32') - height, width = max(inner_size, im.shape[0]), max(inner_size, im.shape[1]) + height, width = max(inner_size, im.shape[0]), max(inner_size, + im.shape[1]) padded_im = np.zeros((height, width)) startY = (height - im.shape[0]) / 2 startX = (width - im.shape[1]) / 2 endY, endX = startY + im.shape[0], startX + im.shape[1] - padded_im[startY: endY, startX: endX] = im + padded_im[startY:endY, startX:endX] = im if test: startY = (height - inner_size) / 2 startX = (width - inner_size) / 2 @@ -72,19 +78,21 @@ def crop_img(im, inner_size, color=True, test=True): startX = np.random.randint(0, width - inner_size + 1) endY, endX = startY + inner_size, startX + inner_size if color: - pic = padded_im[:, startY: endY, startX: endX] + pic = padded_im[:, startY:endY, startX:endX] else: - pic = padded_im[startY: endY, startX: endX] + pic = padded_im[startY:endY, startX:endX] if (not test) and (np.random.randint(2) == 0): pic = flip(pic) return pic + def decode_jpeg(jpeg_string): np_array = np.array(Image.open(StringIO(jpeg_string))) if len(np_array.shape) == 3: np_array = np.transpose(np_array, (2, 0, 1)) return np_array + def preprocess_img(im, img_mean, crop_size, is_train, color=True): """ Does data augmentation for images. @@ -99,6 +107,7 @@ def preprocess_img(im, img_mean, crop_size, is_train, color=True): pic -= img_mean return pic.flatten() + def load_meta(meta_path, mean_img_size, crop_size, color=True): """ Return the loaded meta file. @@ -109,17 +118,18 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True): mean = np.load(meta_path)['data_mean'] border = (mean_img_size - crop_size) / 2 if color: - assert(mean_img_size * mean_img_size * 3 == mean.shape[0]) + assert (mean_img_size * mean_img_size * 3 == mean.shape[0]) mean = mean.reshape(3, mean_img_size, mean_img_size) - mean = mean[:, border: border + crop_size, - border: border + crop_size].astype('float32') + mean = mean[:, border:border + crop_size, border:border + + crop_size].astype('float32') else: - assert(mean_img_size * mean_img_size == mean.shape[0]) + assert (mean_img_size * mean_img_size == mean.shape[0]) mean = mean.reshape(mean_img_size, mean_img_size) - mean = mean[border: border + crop_size, - border: border + crop_size].astype('float32') + mean = mean[border:border + crop_size, border:border + + crop_size].astype('float32') return mean + def load_image(img_path, is_color=True): """ Load image and return. @@ -130,6 +140,7 @@ def load_image(img_path, is_color=True): img.load() return img + def oversample(img, crop_dims): """ image : iterable of (H x W x K) ndarrays @@ -152,50 +163,53 @@ def oversample(img, crop_dims): for j in w_indices: crops_ix[curr] = (i, j, i + crop_dims[0], j + crop_dims[1]) curr += 1 - crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate([ - -crop_dims / 2.0, - crop_dims / 2.0 - ]) + crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate( + [-crop_dims / 2.0, crop_dims / 2.0]) crops_ix = np.tile(crops_ix, (2, 1)) # Extract crops - crops = np.empty((10 * len(img), crop_dims[0], crop_dims[1], - im_shape[-1]), dtype=np.float32) + crops = np.empty( + (10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), + dtype=np.float32) ix = 0 for im in img: for crop in crops_ix: crops[ix] = im[crop[0]:crop[2], crop[1]:crop[3], :] ix += 1 - crops[ix-5:ix] = crops[ix-5:ix, :, ::-1, :] # flip for mirrors + crops[ix - 5:ix] = crops[ix - 5:ix, :, ::-1, :] # flip for mirrors return crops + class ImageTransformer: - def __init__(self, transpose = None, - channel_swap = None, mean = None, is_color = True): + def __init__(self, + transpose=None, + channel_swap=None, + mean=None, + is_color=True): self.transpose = transpose self.channel_swap = None self.mean = None - self.is_color = is_color + self.is_color = is_color - def set_transpose(self, order): + def set_transpose(self, order): if self.is_color: - assert 3 == len(order) + assert 3 == len(order) self.transpose = order - def set_channel_swap(self, order): + def set_channel_swap(self, order): if self.is_color: - assert 3 == len(order) + assert 3 == len(order) self.channel_swap = order def set_mean(self, mean): # mean value, may be one value per channel if mean.ndim == 1: - mean = mean[:, np.newaxis, np.newaxis] - else: + mean = mean[:, np.newaxis, np.newaxis] + else: # elementwise mean if self.is_color: assert len(mean.shape) == 3 - self.mean = mean + self.mean = mean def transformer(self, data): if self.transpose is not None: diff --git a/python/paddle/utils/make_model_diagram.py b/python/paddle/utils/make_model_diagram.py index c8990bf73e8e46ffbb7c5fa4ebe84efe44c75498..29e271717d7108f343d4c28d51c7dfb11bb33fba 100644 --- a/python/paddle/utils/make_model_diagram.py +++ b/python/paddle/utils/make_model_diagram.py @@ -15,7 +15,6 @@ # Generate dot diagram file for the given paddle model config # The generated file can be viewed using Graphviz (http://graphviz.org) - import sys import traceback @@ -46,16 +45,16 @@ def make_diagram(config_file, dot_file, config_arg_str): submodel_layers = set() def make_link(link): - return 'l%s -> l%s;' % ( - name2id[link.layer_name], name2id[link.link_name]) + return 'l%s -> l%s;' % (name2id[link.layer_name], + name2id[link.link_name]) def make_mem(mem): s = '' if mem.boot_layer_name: - s += 'l%s -> l%s;\n' % ( - name2id[mem.boot_layer_name], name2id[mem.layer_name]) - s += 'l%s -> l%s [style=dashed];' % ( - name2id[mem.layer_name], name2id[mem.link_name]) + s += 'l%s -> l%s;\n' % (name2id[mem.boot_layer_name], + name2id[mem.layer_name]) + s += 'l%s -> l%s [style=dashed];' % (name2id[mem.layer_name], + name2id[mem.link_name]) return s print >> f, 'digraph graphname {' @@ -110,8 +109,8 @@ def make_diagram(config_file, dot_file, config_arg_str): def usage(): - print >> sys.stderr, ("Usage: python show_model_diagram.py" - + " CONFIG_FILE DOT_FILE [config_str]") + print >> sys.stderr, ("Usage: python show_model_diagram.py" + + " CONFIG_FILE DOT_FILE [config_str]") exit(1) diff --git a/python/paddle/utils/plotcurve.py b/python/paddle/utils/plotcurve.py index 076eb6f74d5b1871a47446152dc32d2d1762f747..7bc7c5f8d243ed4cca834c48197c511e44baf215 100644 --- a/python/paddle/utils/plotcurve.py +++ b/python/paddle/utils/plotcurve.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Plot training and testing curve from paddle log. It takes input from a file or stdin, and output to a file or stdout. @@ -59,8 +58,8 @@ import re import os -def plot_paddle_curve(keys, inputfile, outputfile, - format='png', show_fig = False): +def plot_paddle_curve(keys, inputfile, outputfile, format='png', + show_fig=False): """Plot curves from paddle log and save to outputfile. :param keys: a list of strings to be plotted, e.g. AvgCost @@ -93,12 +92,17 @@ def plot_paddle_curve(keys, inputfile, outputfile, return m = len(keys) + 1 for i in xrange(1, m): - pyplot.plot(x[:, 0], x[:, i], color=cm.jet(1.0 * (i - 1) / (2 * m)), - label=keys[i - 1]) + pyplot.plot( + x[:, 0], + x[:, i], + color=cm.jet(1.0 * (i - 1) / (2 * m)), + label=keys[i - 1]) if (x_test.shape[0] > 0): - pyplot.plot(x[:, 0], x_test[:, i], - color=cm.jet(1.0 - 1.0 * (i - 1) / (2 * m)), - label="Test " + keys[i - 1]) + pyplot.plot( + x[:, 0], + x_test[:, i], + color=cm.jet(1.0 - 1.0 * (i - 1) / (2 * m)), + label="Test " + keys[i - 1]) pyplot.xlabel('number of epoch') pyplot.legend(loc='best') if show_fig: @@ -111,12 +115,20 @@ def main(argv): """ main method of plotting curves. """ - cmdparser = argparse.ArgumentParser("Plot training and testing curves from paddle log file.") - cmdparser.add_argument('key', nargs='*', help='keys of scores to plot, the default is AvgCost') - cmdparser.add_argument('-i', '--input', help='input filename of paddle log, ' - 'default will be standard input') - cmdparser.add_argument('-o', '--output', help='output filename of figure, ' - 'default will be standard output') + cmdparser = argparse.ArgumentParser( + "Plot training and testing curves from paddle log file.") + cmdparser.add_argument( + 'key', nargs='*', help='keys of scores to plot, the default is AvgCost') + cmdparser.add_argument( + '-i', + '--input', + help='input filename of paddle log, ' + 'default will be standard input') + cmdparser.add_argument( + '-o', + '--output', + help='output filename of figure, ' + 'default will be standard output') cmdparser.add_argument('--format', help='figure format(png|pdf|ps|eps|svg)') args = cmdparser.parse_args(argv) keys = args.key diff --git a/python/paddle/utils/predefined_net.py b/python/paddle/utils/predefined_net.py index 6b0db0b6984d0e139f701f4363291b9c9dd66f82..e9033432ed5200a88ce1ce4f3d7e74d03cf8c8e5 100644 --- a/python/paddle/utils/predefined_net.py +++ b/python/paddle/utils/predefined_net.py @@ -41,9 +41,8 @@ def image_data(data_dir, the size of the mean image, the number of classes. async_load_data: whether to load image data asynchronuously. """ - data_creator = ImageClassificationDatasetCreater(data_dir, - processed_image_size, - color) + data_creator = ImageClassificationDatasetCreater( + data_dir, processed_image_size, color) batch_data_dir = data_dir train_list = os.path.join(batch_data_dir, train_list) test_list = os.path.join(batch_data_dir, test_list) @@ -64,13 +63,17 @@ def image_data(data_dir, 'color': color_string } - define_py_data_sources2(train_list, test_list, - module='image_provider', - obj='processData', - args=args) - return {"image_size": image_size, - "num_classes": num_classes, - "is_color": is_color} + define_py_data_sources2( + train_list, + test_list, + module='image_provider', + obj='processData', + args=args) + return { + "image_size": image_size, + "num_classes": num_classes, + "is_color": is_color + } def get_extra_layer_attr(drop_rate): @@ -80,8 +83,8 @@ def get_extra_layer_attr(drop_rate): return ExtraLayerAttribute(drop_rate=drop_rate) -def image_data_layers(image_size, num_classes, - is_color=False, is_predict=False): +def image_data_layers(image_size, num_classes, is_color=False, + is_predict=False): """ Data layers for image classification. image_size: image size. @@ -109,56 +112,58 @@ def simple_conv_net(data_conf, is_color=False): num_classes: num of classes. is_color: whether the input images are color. """ - for k, v in data_conf.iteritems(): globals()[k] = v + for k, v in data_conf.iteritems(): + globals()[k] = v data_input, label_input, num_image_channels = \ image_data_layers(image_size, num_classes, is_color, is_predict) filter_sizes = [5, 5] num_channels = [32, 64] strides = [1, 1] fc_dims = [500] - conv_bn_pool1 = img_conv_bn_pool(name="g1", - input=data_input, - filter_size=filter_sizes[0], - num_channel=num_image_channels, - num_filters=num_channels[0], - conv_stride=1, - conv_padding=0, - pool_size=3, - pool_stride=2, - act=ReluActivation()) - conv_bn_pool2 = img_conv_bn_pool(name="g2", - input=conv_bn_pool1, - filter_size=filter_sizes[1], - num_channel=num_channels[0], - num_filters=num_channels[1], - conv_stride=1, - conv_padding=0, - pool_size=3, - pool_stride=2, - act=ReluActivation()) - fc3 = fc_layer(name="fc3", - input=conv_bn_pool2, - dim=fc_dims[0], - act=ReluActivation()) - fc3_dropped = dropout_layer(name="fc3_dropped", - input=fc3, - dropout_rate=0.5) - output = fc_layer(name="output", - input=fc3_dropped, - dim=fc_dims[0], - act=SoftmaxActivation()) + conv_bn_pool1 = img_conv_bn_pool( + name="g1", + input=data_input, + filter_size=filter_sizes[0], + num_channel=num_image_channels, + num_filters=num_channels[0], + conv_stride=1, + conv_padding=0, + pool_size=3, + pool_stride=2, + act=ReluActivation()) + conv_bn_pool2 = img_conv_bn_pool( + name="g2", + input=conv_bn_pool1, + filter_size=filter_sizes[1], + num_channel=num_channels[0], + num_filters=num_channels[1], + conv_stride=1, + conv_padding=0, + pool_size=3, + pool_stride=2, + act=ReluActivation()) + fc3 = fc_layer( + name="fc3", input=conv_bn_pool2, dim=fc_dims[0], act=ReluActivation()) + fc3_dropped = dropout_layer(name="fc3_dropped", input=fc3, dropout_rate=0.5) + output = fc_layer( + name="output", + input=fc3_dropped, + dim=fc_dims[0], + act=SoftmaxActivation()) if is_predict: end_of_network(output) else: - cost = classify(name="cost", - input=output, - label=label_input) + cost = classify(name="cost", input=output, label=label_input) end_of_network(cost) -def conv_layer_group(prefix_num, num_layers, input, - input_channels, output_channels, - drop_rates=[], strides=[], +def conv_layer_group(prefix_num, + num_layers, + input, + input_channels, + output_channels, + drop_rates=[], + strides=[], with_bn=[]): """ A set of convolution layers, and batch normalization layers, @@ -190,36 +195,45 @@ def conv_layer_group(prefix_num, num_layers, input, i_conv_in = group_output i_channels_conv = input_channels if i == 1 else output_channels conv_act = LinearActivation() if with_bn[i - 1] else ReluActivation() - conv_output = img_conv_layer(name="conv%d_%d" % (prefix_num, i), - input=i_conv_in, - filter_size=3, - num_channels=i_channels_conv, - num_filters=output_channels, - stride=strides[i - 1], - padding=1, - act=conv_act) + conv_output = img_conv_layer( + name="conv%d_%d" % (prefix_num, i), + input=i_conv_in, + filter_size=3, + num_channels=i_channels_conv, + num_filters=output_channels, + stride=strides[i - 1], + padding=1, + act=conv_act) if with_bn[i - 1]: - bn = batch_norm_layer(name="conv%d_%d_bn" % (prefix_num, i), - input=conv_output, - num_channels=output_channels, - act=ReluActivation(), - layer_attr=get_extra_layer_attr( - drop_rate=drop_rates[i - 1])) + bn = batch_norm_layer( + name="conv%d_%d_bn" % (prefix_num, i), + input=conv_output, + num_channels=output_channels, + act=ReluActivation(), + layer_attr=get_extra_layer_attr(drop_rate=drop_rates[i - 1])) group_output = bn else: group_output = conv_output - pool = img_pool_layer(name="pool%d" % prefix_num, - input=group_output, - pool_size=2, - num_channels=output_channels, - stride=2) + pool = img_pool_layer( + name="pool%d" % prefix_num, + input=group_output, + pool_size=2, + num_channels=output_channels, + stride=2) return pool -def vgg_conv_net(image_size, num_classes, num_layers, - channels, strides, with_bn, fc_dims, - drop_rates, drop_rates_fc=[], - is_color=True, is_predict=False): +def vgg_conv_net(image_size, + num_classes, + num_layers, + channels, + strides, + with_bn, + fc_dims, + drop_rates, + drop_rates_fc=[], + is_color=True, + is_predict=False): """ A Wrapper for a VGG network for image classification. It is a set of convolutional groups followed by several fully @@ -248,51 +262,49 @@ def vgg_conv_net(image_size, num_classes, num_layers, for i in range(len(num_layers)): input_layer = data_input if i == 0 else group_output input_channels = 3 if i == 0 else channels[i - 1] - group_output = conv_layer_group(prefix_num=i + 1, - num_layers=num_layers[i], - input=input_layer, - input_channels=input_channels, - output_channels=channels[i], - drop_rates=drop_rates[i], - strides=strides[i], - with_bn=with_bn[i]) + group_output = conv_layer_group( + prefix_num=i + 1, + num_layers=num_layers[i], + input=input_layer, + input_channels=input_channels, + output_channels=channels[i], + drop_rates=drop_rates[i], + strides=strides[i], + with_bn=with_bn[i]) conv_output_name = group_output if drop_rates_fc[0] != 0.0: dropped_pool_name = "pool_dropped" - conv_output_name = dropout_layer(name=dropped_pool_name, - input=conv_output_name, - dropout_rate=drop_rates_fc[0]) + conv_output_name = dropout_layer( + name=dropped_pool_name, + input=conv_output_name, + dropout_rate=drop_rates_fc[0]) for i in range(len(fc_dims)): input_layer_name = conv_output_name if i == 0 else fc_output active_type = LinearActivation() if i == len( fc_dims) - 1 else ReluActivation() drop_rate = 0.0 if i == len(fc_dims) - 1 else drop_rates_fc[i + 1] - fc_output = fc_layer(name="fc%d" % (i + 1), - input=input_layer_name, - size=fc_dims[i], - act=active_type, - layer_attr=get_extra_layer_attr(drop_rate)) - bn = batch_norm_layer(name="fc_bn", - input=fc_output, - num_channels=fc_dims[len(fc_dims) - 1], - act=ReluActivation(), - layer_attr=get_extra_layer_attr( - drop_rate=drop_rates_fc[-1])) - output = fc_layer(name="output", - input=bn, - size=num_classes, - act=SoftmaxActivation()) + fc_output = fc_layer( + name="fc%d" % (i + 1), + input=input_layer_name, + size=fc_dims[i], + act=active_type, + layer_attr=get_extra_layer_attr(drop_rate)) + bn = batch_norm_layer( + name="fc_bn", + input=fc_output, + num_channels=fc_dims[len(fc_dims) - 1], + act=ReluActivation(), + layer_attr=get_extra_layer_attr(drop_rate=drop_rates_fc[-1])) + output = fc_layer( + name="output", input=bn, size=num_classes, act=SoftmaxActivation()) if is_predict: outputs(output) else: - cost = classification_cost(name="cost", - input=output, - label=label_input) + cost = classification_cost(name="cost", input=output, label=label_input) outputs(cost) -def vgg16_conv_net(image_size, num_classes, - is_color=True, is_predict=False): +def vgg16_conv_net(image_size, num_classes, is_color=True, is_predict=False): """ A Wrapper for a 16 layers VGG network for image classification. The detailed architecture of the paper can be found here: @@ -314,8 +326,7 @@ def vgg16_conv_net(image_size, num_classes, is_predict=is_predict) -def small_vgg(data_conf, - is_predict=False): +def small_vgg(data_conf, is_predict=False): """ A Wrapper for a small VGG network for CIFAR-10 image classification. The detailed architecture of the paper can be found here: @@ -329,7 +340,8 @@ def small_vgg(data_conf, num_classes: num of classes. is_color: whether the input images are color. """ - for k, v in data_conf.iteritems(): globals()[k] = v + for k, v in data_conf.iteritems(): + globals()[k] = v vgg_conv_net(image_size, num_classes, num_layers=[2, 2, 3, 3], channels=[64, 128, 256, 512], @@ -343,8 +355,11 @@ def small_vgg(data_conf, is_predict=is_predict) -def training_settings(learning_rate=0.1, batch_size=128, algorithm="sgd", - momentum=0.9, decay_rate=0.001): +def training_settings(learning_rate=0.1, + batch_size=128, + algorithm="sgd", + momentum=0.9, + decay_rate=0.001): """ Training settings. learning_rate: learning rate of the training. @@ -357,8 +372,9 @@ def training_settings(learning_rate=0.1, batch_size=128, algorithm="sgd", momentum: momentum of the training algorithm. decay_rate: weight decay rate. """ - Settings(algorithm=algorithm, - batch_size=batch_size, - learning_rate=learning_rate / float(batch_size)) + Settings( + algorithm=algorithm, + batch_size=batch_size, + learning_rate=learning_rate / float(batch_size)) default_momentum(momentum) default_decay_rate(decay_rate * batch_size) diff --git a/python/paddle/utils/preprocess_img.py b/python/paddle/utils/preprocess_img.py index 2765e93fa14e210c5ec65d5cfac4cd57a793927a..f3c609e4cd1a3714219965cd543ab11136d3585f 100644 --- a/python/paddle/utils/preprocess_img.py +++ b/python/paddle/utils/preprocess_img.py @@ -28,16 +28,18 @@ def resize_image(img, target_size): img: the input image to be resized. target_size: the target resized image size. """ - percent = (target_size/float(min(img.size[0], img.size[1]))) + percent = (target_size / float(min(img.size[0], img.size[1]))) resized_size = int(round(img.size[0] * percent)),\ int(round(img.size[1] * percent)) img = img.resize(resized_size, Image.ANTIALIAS) return img + class DiskImage: """ A class of image data on disk. """ + def __init__(self, path, target_size): """ path: path of the image. @@ -77,6 +79,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): """ A class to process data for image classification. """ + def __init__(self, data_path, target_size, color=True): """ data_path: the path to store the training data and batches. @@ -95,8 +98,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): The meta file contains the meam image, as well as some configs. data: the training Dataaet. """ - output_path = os.path.join(self.data_path, - self.batch_dir_name, + output_path = os.path.join(self.data_path, self.batch_dir_name, self.meta_filename) if self.color: mean_img = np.zeros((3, self.target_size, self.target_size)) @@ -108,12 +110,13 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): mean_img += cropped_img mean_img /= len(data.data) mean_img = mean_img.astype('int32').flatten() - preprocess_util.save_file({"data_mean": mean_img, - "image_size": self.target_size, - "mean_image_size": self.target_size, - "num_classes": self.num_classes, - "color": self.color}, - output_path) + preprocess_util.save_file({ + "data_mean": mean_img, + "image_size": self.target_size, + "mean_image_size": self.target_size, + "num_classes": self.num_classes, + "color": self.color + }, output_path) pass def create_dataset_from_list(self, path): @@ -125,12 +128,11 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): label_name = items[1] if not label_name in label_set: label_set[label_name] = len(label_set.keys()) - img = DiskImage(path = image_path, target_size = self.target_size) - label = preprocess_util.Lablel(label = label_set[label_name], - name=label_name) + img = DiskImage(path=image_path, target_size=self.target_size) + label = preprocess_util.Lablel( + label=label_set[label_name], name=label_name) return preprocess_util.Dataset(data, self.keys), label_set - def create_dataset_from_dir(self, path): """ Create a Dataset object for image classfication. @@ -143,11 +145,12 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): label_set = preprocess_util.get_label_set_from_dir(path) data = [] for l_name in label_set.keys(): - image_paths = preprocess_util.list_images(os.path.join(path, l_name)) + image_paths = preprocess_util.list_images( + os.path.join(path, l_name)) for p in image_paths: - img = DiskImage(path = p, target_size = self.target_size) - label = preprocess_util.Label(label = label_set[l_name], - name = l_name) + img = DiskImage(path=p, target_size=self.target_size) + label = preprocess_util.Label( + label=label_set[l_name], name=l_name) data.append((img, label)) random.shuffle(data) return preprocess_util.Dataset(data, self.keys), label_set diff --git a/python/paddle/utils/preprocess_util.py b/python/paddle/utils/preprocess_util.py index f187ed1f67366ab9d6c74a9b5f030b7861168879..e5067a80ea7005bee7781e885b3658a2c03dc6f2 100644 --- a/python/paddle/utils/preprocess_util.py +++ b/python/paddle/utils/preprocess_util.py @@ -18,6 +18,7 @@ import cPickle as pickle import random import collections + def save_file(data, filename): """ Save data into pickle format. @@ -26,6 +27,7 @@ def save_file(data, filename): """ pickle.dump(data, open(filename, 'wb'), protocol=pickle.HIGHEST_PROTOCOL) + def save_list(l, outfile): """ Save a list of string into a text file. There is one line for each string. @@ -42,15 +44,20 @@ def exclude_pattern(f): """ return f.startswith(".") or f.endswith("~") + def list_dirs(path): """ Return a list of directories in path. Exclude all the directories that start with '.'. path: the base directory to search over. """ - return [os.path.join(path, d) for d in next(os.walk(path))[1] if not exclude_pattern(d)] + return [ + os.path.join(path, d) for d in next(os.walk(path))[1] + if not exclude_pattern(d) + ] -def list_images(path, exts = set(["jpg", "png", "bmp", "jpeg"])): + +def list_images(path, exts=set(["jpg", "png", "bmp", "jpeg"])): """ Return a list of images in path. path: the base directory to search over. @@ -60,6 +67,7 @@ def list_images(path, exts = set(["jpg", "png", "bmp", "jpeg"])): if os.path.isfile(os.path.join(path, d)) and not exclude_pattern(d)\ and os.path.splitext(d)[-1][1:] in exts] + def list_files(path): """ Return a list of files in path. @@ -69,6 +77,7 @@ def list_files(path): return [os.path.join(path, d) for d in os.listdir(path) \ if os.path.isfile(os.path.join(path, d)) and not exclude_pattern(d)] + def get_label_set_from_dir(path): """ Return a dictionary of the labels and label ids from a path. @@ -84,6 +93,7 @@ class Label: """ A class of label data. """ + def __init__(self, label, name): """ label: the id of the label. @@ -98,9 +108,10 @@ class Label: """ return int(self.label) - def __hash__(self): + def __hash__(self): return hash((self.label)) + class Dataset: """ A class to represent a dataset. A dataset contains a set of items. @@ -108,6 +119,7 @@ class Dataset: For example: in image classification dataset, each item contains two slot, The first slot is an image, and the second slot is a label. """ + def __init__(self, data, keys): """ data: a list of data. @@ -120,7 +132,7 @@ class Dataset: def check_valid(self): for d in self.data: - assert(len(d) == len(self.keys)) + assert (len(d) == len(self.keys)) def permute(self, key_id, num_per_batch): """ @@ -167,8 +179,9 @@ class Dataset: while len(permuted_data) < len(self.data): for k in keyvalue_indices: begin_idx = keyvalue_readpointer[k] - end_idx = int(min(begin_idx + num_data_per_key_batch, - len(keyvalue_indices[k]))) + end_idx = int( + min(begin_idx + num_data_per_key_batch, + len(keyvalue_indices[k]))) print "begin_idx, end_idx" print begin_idx, end_idx for idx in range(begin_idx, end_idx): @@ -177,12 +190,12 @@ class Dataset: self.data = permuted_data - class DataBatcher: """ A class that is used to create batches for both training and testing datasets. """ + def __init__(self, train_data, test_data, label_set): """ train_data, test_data: Each one is a dataset object repesenting @@ -190,10 +203,10 @@ class DataBatcher: label_set: a dictionary storing the mapping from label name to label id. """ self.train_data = train_data - self.test_data = test_data + self.test_data = test_data self.label_set = label_set self.num_per_batch = 5000 - assert(self.train_data.keys == self.test_data.keys) + assert (self.train_data.keys == self.test_data.keys) def create_batches_and_list(self, output_path, train_list_name, test_list_name, label_set_name): @@ -202,16 +215,19 @@ class DataBatcher: It also create train.list and test.list to indicate the list of the batch files for training and testing data, respectively. """ - train_list = self.create_batches(self.train_data, output_path, - "train_", self.num_per_batch) + train_list = self.create_batches(self.train_data, output_path, "train_", + self.num_per_batch) test_list = self.create_batches(self.test_data, output_path, "test_", - self.num_per_batch) + self.num_per_batch) save_list(train_list, os.path.join(output_path, train_list_name)) save_list(test_list, os.path.join(output_path, test_list_name)) save_file(self.label_set, os.path.join(output_path, label_set_name)) - def create_batches(self, data, output_path, - prefix = "", num_data_per_batch=5000): + def create_batches(self, + data, + output_path, + prefix="", + num_data_per_batch=5000): """ Create batches for a Dataset object. data: the Dataset object to process. @@ -244,6 +260,7 @@ class DatasetCreater(object): - create_dataset() - create_meta_file() """ + def __init__(self, data_path): """ data_path: the path to store the training data and batches. @@ -324,24 +341,22 @@ class DatasetCreater(object): out_path = os.path.join(self.data_path, self.batch_dir_name) if not os.path.exists(out_path): os.makedirs(out_path) - if (self.overwrite or - not os.path.exists(os.path.join(out_path, self.train_list_name))): + if (self.overwrite or not os.path.exists( + os.path.join(out_path, self.train_list_name))): train_data, train_label_set = \ self.create_dataset(train_path) test_data, test_label_set = \ self.create_dataset(test_path) - train_data.permute(self.keys.index(self.permutate_key), - self.num_per_batch) + train_data.permute( + self.keys.index(self.permutate_key), self.num_per_batch) - assert(train_label_set == test_label_set) - data_batcher = DataBatcher(train_data, test_data, - train_label_set) + assert (train_label_set == test_label_set) + data_batcher = DataBatcher(train_data, test_data, train_label_set) data_batcher.num_per_batch = self.num_per_batch - data_batcher.create_batches_and_list(self.output_path, - self.train_list_name, - self.test_list_name, - self.label_set_name) + data_batcher.create_batches_and_list( + self.output_path, self.train_list_name, self.test_list_name, + self.label_set_name) self.num_classes = len(train_label_set.keys()) self.create_meta_file(train_data) return out_path diff --git a/python/paddle/utils/show_pb.py b/python/paddle/utils/show_pb.py index 81b28c48554efb3618baf24f7e62a213f3b5f2e2..3b371727b84e51f9f7db80b34e6e38fd149fcaaa 100644 --- a/python/paddle/utils/show_pb.py +++ b/python/paddle/utils/show_pb.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Show the content of proto buffer data file of PADDLE """ @@ -21,6 +20,7 @@ import sys from google.protobuf.internal.decoder import _DecodeVarint import paddle.proto.DataFormat_pb2 as DataFormat + def read_proto(file, message): """ read a protobuffer struct from file, the length of the struct is stored as @@ -39,7 +39,7 @@ def read_proto(file, message): def usage(): - print >>sys.stderr, "Usage: python show_pb.py PROTO_DATA_FILE" + print >> sys.stderr, "Usage: python show_pb.py PROTO_DATA_FILE" exit(1) @@ -51,10 +51,7 @@ if __name__ == '__main__': header = DataFormat.DataHeader() read_proto(f, header) print header - + sample = DataFormat.DataSample() while read_proto(f, sample): print sample - - - diff --git a/python/paddle/utils/torch2paddle.py b/python/paddle/utils/torch2paddle.py index 6a5ea130615e7add6a8b2999a7c365836653d8e0..958f55dbc4ee0b588b78fb630153b585f1ad4be0 100644 --- a/python/paddle/utils/torch2paddle.py +++ b/python/paddle/utils/torch2paddle.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Convert torch parameter file to paddle model files. @@ -28,10 +27,11 @@ import torchfile import cPickle as pickle import argparse + # save parameters def save_layer_parameters(outfile, feats): version = 0 - value_size = 4; + value_size = 4 ret = "" for feat in feats: ret += feat.tostring() @@ -41,16 +41,18 @@ def save_layer_parameters(outfile, feats): fo.write(ret) fo.close() + def save_net_parameters(layers, params, output_path): for i in range(len(layers)): - weight = params[i*2] - biases = params[i*2+1] + weight = params[i * 2] + biases = params[i * 2 + 1] weight_file = os.path.join(output_path, '_%s.w0' % layers[i]) biases_file = os.path.join(output_path, '_%s.wbias' % layers[i]) print "Saving for layer %s." % layers[i] save_layer_parameters(weight_file, [weight]) save_layer_parameters(biases_file, biases) + def load_layer_parameters(filename): fn = open(filename, 'rb') version, = struct.unpack('i', fn.read(4)) @@ -60,16 +62,20 @@ def load_layer_parameters(filename): value = np.fromfile(fn, dtype) return value + def main(argv): """ main method of converting torch to paddle files. :param argv: :return: """ - cmdparser = argparse.ArgumentParser("Convert torch parameter file to paddle model files.") - cmdparser.add_argument('-i', '--input', help='input filename of torch parameters') + cmdparser = argparse.ArgumentParser( + "Convert torch parameter file to paddle model files.") + cmdparser.add_argument( + '-i', '--input', help='input filename of torch parameters') cmdparser.add_argument('-l', '--layers', help='list of layer names') - cmdparser.add_argument('-o', '--output', help='output file path of paddle model') + cmdparser.add_argument( + '-o', '--output', help='output file path of paddle model') args = cmdparser.parse_args(argv) if args.input and args.layers and args.output: @@ -77,7 +83,10 @@ def main(argv): layers = [line.strip() for line in open(args.layers, 'r')] save_net_parameters(layers, params, args.output) else: - print('Usage: python torch2paddle.py -i torchfile.t7 -l layers.txt -o path/to/paddle_model') + print( + 'Usage: python torch2paddle.py -i torchfile.t7 -l layers.txt -o path/to/paddle_model' + ) + if __name__ == "__main__": main(sys.argv[1:])