diff --git a/README.md b/README.md index e8679fb55fc22559d933a416e8706b7baf536ead..bd47ed44bc808196b0e6598f28d72620422f3e1a 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ # PaddlePaddle -[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle) -[![Coverage Status](https://coveralls.io/repos/github/baidu/Paddle/badge.svg?branch=develop)](https://coveralls.io/github/baidu/Paddle?branch=develop) -[![Join the chat at https://gitter.im/PaddlePaddle/Deep_Learning](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/PaddlePaddle/Deep_Learning?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE) +[![Build Status](https://travis-ci.org/PaddlePaddle/Paddle.svg?branch=develop)](https://travis-ci.org/baidu/Paddle) +[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://www.paddlepaddle.org/) +[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://www.paddlepaddle.org/cn/index.html) +[![Coverage Status](https://coveralls.io/repos/github/PaddlePaddle/Paddle/badge.svg?branch=develop)](https://coveralls.io/github/baidu/Paddle?branch=develop) +[![Release](https://img.shields.io/github/release/baidu/Paddle.svg?colorB=fedcba)](https://github.com/baidu/Paddle/releases) +[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE) + Welcome to the PaddlePaddle GitHub. diff --git a/demo/semantic_role_labeling/data/extract_dict_feature.py b/demo/semantic_role_labeling/data/extract_dict_feature.py index 2982e54c665b41400aab0a893ff3c76335404988..daca5f01cf2b3bd231bf530f17ec760272ce93e0 100644 --- a/demo/semantic_role_labeling/data/extract_dict_feature.py +++ b/demo/semantic_role_labeling/data/extract_dict_feature.py @@ -17,24 +17,15 @@ import os from optparse import OptionParser -def extract_dict_features(pair_file, feature_file, src_dict_file, - tgt_dict_file): - src_dict = set() - tgt_dict = set() - - with open(pair_file) as fin, open(feature_file, 'w') as feature_out, open( - src_dict_file, 'w') as src_dict_out, open(tgt_dict_file, - 'w') as tgt_dict_out: +def extract_dict_features(pair_file, feature_file): + + with open(pair_file) as fin, open(feature_file, 'w') as feature_out: for line in fin: - sentence, labels = line.strip().split('\t') + sentence, predicate, labels = line.strip().split('\t') sentence_list = sentence.split() labels_list = labels.split() - src_dict.update(sentence_list) - tgt_dict.update(labels_list) - verb_index = labels_list.index('B-V') - verb_feature = sentence_list[verb_index] mark = [0] * len(labels_list) if verb_index > 0: @@ -42,47 +33,50 @@ def extract_dict_features(pair_file, feature_file, src_dict_file, ctx_n1 = sentence_list[verb_index - 1] else: ctx_n1 = 'bos' - ctx_n1_feature = ctx_n1 + + if verb_index > 1: + mark[verb_index - 2] = 1 + ctx_n2 = sentence_list[verb_index - 2] + else: + ctx_n2 = 'bos' mark[verb_index] = 1 - ctx_0_feature = sentence_list[verb_index] + ctx_0 = sentence_list[verb_index] if verb_index < len(labels_list) - 2: mark[verb_index + 1] = 1 ctx_p1 = sentence_list[verb_index + 1] else: ctx_p1 = 'eos' - ctx_p1_feature = ctx_p1 + + if verb_index < len(labels_list) - 3: + mark[verb_index + 2] = 1 + ctx_p2 = sentence_list[verb_index + 2] + else: + ctx_p2 = 'eos' + feature_str = sentence + '\t' \ - + verb_feature + '\t' \ - + ctx_n1_feature + '\t' \ - + ctx_0_feature + '\t' \ - + ctx_p1_feature + '\t' \ + + predicate + '\t' \ + + ctx_n2 + '\t' \ + + ctx_n1 + '\t' \ + + ctx_0 + '\t' \ + + ctx_p1 + '\t' \ + + ctx_p2 + '\t' \ + ' '.join([str(i) for i in mark]) + '\t' \ + labels feature_out.write(feature_str + '\n') - src_dict_out.write('\n') - src_dict_out.write('\n'.join(list(src_dict))) - - tgt_dict_out.write('\n'.join(list(tgt_dict))) if __name__ == '__main__': - usage = '-p pair_file -f feature_file -s source dictionary -t target dictionary ' + usage = '-p pair_file -f feature_file' parser = OptionParser(usage) parser.add_option('-p', dest='pair_file', help='the pair file') - parser.add_option( - '-f', dest='feature_file', help='the file to store feature') - parser.add_option( - '-s', dest='src_dict', help='the file to store source dictionary') - parser.add_option( - '-t', dest='tgt_dict', help='the file to store target dictionary') + parser.add_option('-f', dest='feature_file', help='the feature file') (options, args) = parser.parse_args() - extract_dict_features(options.pair_file, options.feature_file, - options.src_dict, options.tgt_dict) + extract_dict_features(options.pair_file, options.feature_file) diff --git a/demo/semantic_role_labeling/data/extract_pairs.py b/demo/semantic_role_labeling/data/extract_pairs.py index 4d1bef8f958a62be9941d474a0b67542dcc5cfab..86ab00ce41723169de035a841d9e129a1b9e82a3 100644 --- a/demo/semantic_role_labeling/data/extract_pairs.py +++ b/demo/semantic_role_labeling/data/extract_pairs.py @@ -51,7 +51,7 @@ def read_sentences(words_file): for line in fin: line = line.strip() if line == '': - sentences.append(s.lower()) + sentences.append(s) s = '' else: s += line + ' ' @@ -64,6 +64,11 @@ def transform_labels(sentences, labels): if len(labels[i]) == 1: continue else: + verb_list = [] + for x in labels[i][0]: + if x !='-': + verb_list.append(x) + for j in xrange(1, len(labels[i])): label_list = labels[i][j] current_tag = 'O' @@ -88,8 +93,7 @@ def transform_labels(sentences, labels): is_in_bracket = True else: print 'error:', ll - - sen_lab_pair.append((sentences[i], label_seq)) + sen_lab_pair.append((sentences[i], verb_list[j-1], label_seq)) return sen_lab_pair @@ -97,9 +101,9 @@ def write_file(sen_lab_pair, output_file): with open(output_file, 'w') as fout: for x in sen_lab_pair: sentence = x[0] - label_seq = ' '.join(x[1]) - assert len(sentence.split()) == len(x[1]) - fout.write(sentence + '\t' + label_seq + '\n') + label_seq = ' '.join(x[2]) + assert len(sentence.split()) == len(x[2]) + fout.write(sentence + '\t' + x[1]+'\t' +label_seq + '\n') if __name__ == '__main__': diff --git a/demo/semantic_role_labeling/data/get_data.sh b/demo/semantic_role_labeling/data/get_data.sh index 268c0995e27006ec62f38bdda9b0a0994dab096c..55e33f4685627ed483aa6642c518a33558091531 100644 --- a/demo/semantic_role_labeling/data/get_data.sh +++ b/demo/semantic_role_labeling/data/get_data.sh @@ -14,6 +14,10 @@ # limitations under the License. set -e wget http://www.cs.upc.edu/~srlconll/conll05st-tests.tar.gz +wget https://www.googledrive.com/host/0B7Q8d52jqeI9ejh6Q1RpMTFQT1k/semantic_role_labeling/verbDict.txt --no-check-certificate +wget https://www.googledrive.com/host/0B7Q8d52jqeI9ejh6Q1RpMTFQT1k/semantic_role_labeling/targetDict.txt --no-check-certificate +wget https://www.googledrive.com/host/0B7Q8d52jqeI9ejh6Q1RpMTFQT1k/semantic_role_labeling/wordDict.txt --no-check-certificate +wget https://www.googledrive.com/host/0B7Q8d52jqeI9ejh6Q1RpMTFQT1k/semantic_role_labeling/emb --no-check-certificate tar -xzvf conll05st-tests.tar.gz rm conll05st-tests.tar.gz cp ./conll05st-release/test.wsj/words/test.wsj.words.gz . @@ -22,4 +26,4 @@ gunzip test.wsj.words.gz gunzip test.wsj.props.gz python extract_pairs.py -w test.wsj.words -p test.wsj.props -o test.wsj.seq_pair -python extract_dict_feature.py -p test.wsj.seq_pair -f feature -s src.dict -t tgt.dict +python extract_dict_feature.py -p test.wsj.seq_pair -f feature diff --git a/demo/semantic_role_labeling/dataprovider.py b/demo/semantic_role_labeling/dataprovider.py index 5c003584a52d459f13b7942ebe3a7147ac58a42f..d4c137ef42c4e2ec609f3e6f809363e602dfd8dd 100644 --- a/demo/semantic_role_labeling/dataprovider.py +++ b/demo/semantic_role_labeling/dataprovider.py @@ -17,11 +17,15 @@ from paddle.trainer.PyDataProvider2 import * UNK_IDX = 0 -def hook(settings, word_dict, label_dict, **kwargs): +def hook(settings, word_dict, label_dict, predicate_dict, **kwargs): settings.word_dict = word_dict settings.label_dict = label_dict + settings.predicate_dict = predicate_dict + #all inputs are integral and sequential type settings.slots = [ + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(predicate_dict)), integer_value_sequence(len(word_dict)), integer_value_sequence(len(word_dict)), integer_value_sequence(len(word_dict)), @@ -31,27 +35,33 @@ def hook(settings, word_dict, label_dict, **kwargs): ] -@provider(init_hook=hook) -def process(obj, file_name): +def get_batch_size(yeild_data): + return len(yeild_data[0]) + + +@provider(init_hook=hook, should_shuffle=True, calc_batch_size=get_batch_size, + can_over_batch_size=False, cache=CacheType.CACHE_PASS_IN_MEM) +def process(settings, file_name): with open(file_name, 'r') as fdata: for line in fdata: - sentence, predicate, ctx_n1, ctx_0, ctx_p1, mark, label = \ + sentence, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, label = \ line.strip().split('\t') - + words = sentence.split() sen_len = len(words) - word_slot = [obj.word_dict.get(w, UNK_IDX) for w in words] + word_slot = [settings.word_dict.get(w, UNK_IDX) for w in words] - predicate_slot = [obj.word_dict.get(predicate, UNK_IDX)] * sen_len - ctx_n1_slot = [obj.word_dict.get(ctx_n1, UNK_IDX)] * sen_len - ctx_0_slot = [obj.word_dict.get(ctx_0, UNK_IDX)] * sen_len - ctx_p1_slot = [obj.word_dict.get(ctx_p1, UNK_IDX)] * sen_len + predicate_slot = [settings.predicate_dict.get(predicate)] * sen_len + ctx_n2_slot = [settings.word_dict.get(ctx_n2, UNK_IDX)] * sen_len + ctx_n1_slot = [settings.word_dict.get(ctx_n1, UNK_IDX)] * sen_len + ctx_0_slot = [settings.word_dict.get(ctx_0, UNK_IDX)] * sen_len + ctx_p1_slot = [settings.word_dict.get(ctx_p1, UNK_IDX)] * sen_len + ctx_p2_slot = [settings.word_dict.get(ctx_p2, UNK_IDX)] * sen_len marks = mark.split() mark_slot = [int(w) for w in marks] label_list = label.split() - label_slot = [obj.label_dict.get(w) for w in label_list] - - yield word_slot, predicate_slot, ctx_n1_slot, \ - ctx_0_slot, ctx_p1_slot, mark_slot, label_slot + label_slot = [settings.label_dict.get(w) for w in label_list] + yield word_slot, predicate_slot, ctx_n2_slot, ctx_n1_slot, \ + ctx_0_slot, ctx_p1_slot, ctx_p2_slot, mark_slot, label_slot diff --git a/demo/semantic_role_labeling/db_lstm.py b/demo/semantic_role_labeling/db_lstm.py index e3f6edad6972112ed04e173a9b714e3fec13d402..54ceff0e724220cc9ea96b9e0ec6844947a8343e 100644 --- a/demo/semantic_role_labeling/db_lstm.py +++ b/demo/semantic_role_labeling/db_lstm.py @@ -18,8 +18,9 @@ import sys from paddle.trainer_config_helpers import * #file paths -word_dict_file = './data/src.dict' -label_dict_file = './data/tgt.dict' +word_dict_file = './data/wordDict.txt' +label_dict_file = './data/targetDict.txt' +predicate_file= './data/verbDict.txt' train_list_file = './data/train.list' test_list_file = './data/test.list' @@ -30,8 +31,10 @@ if not is_predict: #load dictionaries word_dict = dict() label_dict = dict() + predicate_dict = dict() with open(word_dict_file, 'r') as f_word, \ - open(label_dict_file, 'r') as f_label: + open(label_dict_file, 'r') as f_label, \ + open(predicate_file, 'r') as f_pre: for i, line in enumerate(f_word): w = line.strip() word_dict[w] = i @@ -40,6 +43,11 @@ if not is_predict: w = line.strip() label_dict[w] = i + for i, line in enumerate(f_pre): + w = line.strip() + predicate_dict[w] = i + + if is_test: train_list_file = None @@ -50,91 +58,157 @@ if not is_predict: module='dataprovider', obj='process', args={'word_dict': word_dict, - 'label_dict': label_dict}) + 'label_dict': label_dict, + 'predicate_dict': predicate_dict }) word_dict_len = len(word_dict) label_dict_len = len(label_dict) + pred_len = len(predicate_dict) else: word_dict_len = get_config_arg('dict_len', int) label_dict_len = get_config_arg('label_len', int) + pred_len = get_config_arg('pred_len', int) +############################## Hyper-parameters ################################## mark_dict_len = 2 word_dim = 32 mark_dim = 5 -hidden_dim = 128 +hidden_dim = 512 depth = 8 -emb_lr = 1e-2 -fc_lr = 1e-2 -lstm_lr = 2e-2 + + + +########################### Optimizer ####################################### + settings( batch_size=150, - learning_method=AdamOptimizer(), - learning_rate=1e-3, + learning_method=MomentumOptimizer(momentum=0), + learning_rate=2e-2, regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25) + is_async=False, + model_average=ModelAverage(average_window=0.5, + max_average_window=10000), + +) -#6 features + + + +####################################### network ############################## +#8 features and 1 target word = data_layer(name='word_data', size=word_dict_len) -predicate = data_layer(name='verb_data', size=word_dict_len) +predicate = data_layer(name='verb_data', size=pred_len) + +ctx_n2 = data_layer(name='ctx_n2_data', size=word_dict_len) ctx_n1 = data_layer(name='ctx_n1_data', size=word_dict_len) ctx_0 = data_layer(name='ctx_0_data', size=word_dict_len) ctx_p1 = data_layer(name='ctx_p1_data', size=word_dict_len) +ctx_p2 = data_layer(name='ctx_p2_data', size=word_dict_len) mark = data_layer(name='mark_data', size=mark_dict_len) + if not is_predict: target = data_layer(name='target', size=label_dict_len) -ptt = ParameterAttribute(name='src_emb', learning_rate=emb_lr) -layer_attr = ExtraLayerAttribute(drop_rate=0.5) -fc_para_attr = ParameterAttribute(learning_rate=fc_lr) -lstm_para_attr = ParameterAttribute(initial_std=0., learning_rate=lstm_lr) -para_attr = [fc_para_attr, lstm_para_attr] -word_embedding = embedding_layer(size=word_dim, input=word, param_attr=ptt) -predicate_embedding = embedding_layer( - size=word_dim, input=predicate, param_attr=ptt) -ctx_n1_embedding = embedding_layer(size=word_dim, input=ctx_n1, param_attr=ptt) -ctx_0_embedding = embedding_layer(size=word_dim, input=ctx_0, param_attr=ptt) -ctx_p1_embedding = embedding_layer(size=word_dim, input=ctx_p1, param_attr=ptt) -mark_embedding = embedding_layer(size=mark_dim, input=mark) +default_std=1/math.sqrt(hidden_dim)/3.0 + +emb_para = ParameterAttribute(name='emb', initial_std=0., learning_rate=0.) +std_0 = ParameterAttribute(initial_std=0.) +std_default = ParameterAttribute(initial_std=default_std) + +predicate_embedding = embedding_layer(size=word_dim, input=predicate, param_attr=ParameterAttribute(name='vemb',initial_std=default_std)) +mark_embedding = embedding_layer(name='word_ctx-in_embedding', size=mark_dim, input=mark, param_attr=std_0) + +word_input=[word, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2] +emb_layers = [embedding_layer(size=word_dim, input=x, param_attr=emb_para) for x in word_input] +emb_layers.append(predicate_embedding) +emb_layers.append(mark_embedding) hidden_0 = mixed_layer( + name='hidden0', size=hidden_dim, - input=[ - full_matrix_projection(input=word_embedding), - full_matrix_projection(input=predicate_embedding), - full_matrix_projection(input=ctx_n1_embedding), - full_matrix_projection(input=ctx_0_embedding), - full_matrix_projection(input=ctx_p1_embedding), - full_matrix_projection(input=mark_embedding), - ]) + bias_attr=std_default, + input=[ full_matrix_projection(input=emb, param_attr=std_default ) for emb in emb_layers ]) + -lstm_0 = lstmemory(input=hidden_0, layer_attr=layer_attr) +mix_hidden_lr = 1e-3 +lstm_para_attr = ParameterAttribute(initial_std=0.0, learning_rate=1.0) +hidden_para_attr = ParameterAttribute(initial_std=default_std, learning_rate=mix_hidden_lr) + +lstm_0 = lstmemory(name='lstm0', + input=hidden_0, + act=ReluActivation(), + gate_act=SigmoidActivation(), + state_act=SigmoidActivation(), + bias_attr=std_0, + param_attr=lstm_para_attr) #stack L-LSTM and R-LSTM with direct edges input_tmp = [hidden_0, lstm_0] + for i in range(1, depth): - fc = fc_layer(input=input_tmp, size=hidden_dim, param_attr=para_attr) + mix_hidden = mixed_layer(name='hidden'+str(i), + size=hidden_dim, + bias_attr=std_default, + input=[full_matrix_projection(input=input_tmp[0], param_attr=hidden_para_attr), + full_matrix_projection(input=input_tmp[1], param_attr=lstm_para_attr) + ] + ) + + lstm = lstmemory(name='lstm'+str(i), + input=mix_hidden, + act=ReluActivation(), + gate_act=SigmoidActivation(), + state_act=SigmoidActivation(), + reverse=((i % 2)==1), + bias_attr=std_0, + param_attr=lstm_para_attr) + + input_tmp = [mix_hidden, lstm] + +feature_out = mixed_layer(name='output', + size=label_dict_len, + bias_attr=std_default, + input=[full_matrix_projection(input=input_tmp[0], param_attr=hidden_para_attr), + full_matrix_projection(input=input_tmp[1], param_attr=lstm_para_attr) + ], + ) - lstm = lstmemory( - input=fc, - act=ReluActivation(), - reverse=(i % 2) == 1, - layer_attr=layer_attr) - input_tmp = [fc, lstm] -prob = fc_layer( - input=input_tmp, - size=label_dict_len, - act=SoftmaxActivation(), - param_attr=para_attr) if not is_predict: - cls = classification_cost(input=prob, label=target) - outputs(cls) + crf_l = crf_layer( name = 'crf', + size = label_dict_len, + input = feature_out, + label = target, + param_attr=ParameterAttribute(name='crfw',initial_std=default_std, learning_rate=mix_hidden_lr) + + ) + + + crf_dec_l = crf_decoding_layer(name = 'crf_dec_l', + size = label_dict_len, + input = feature_out, + label = target, + param_attr=ParameterAttribute(name='crfw') + ) + + + eval = sum_evaluator(input=crf_dec_l) + + outputs(crf_l) + else: - outputs(prob) + crf_dec_l = crf_decoding_layer(name = 'crf_dec_l', + size = label_dict_len, + input = feature_out, + param_attr=ParameterAttribute(name='crfw') + ) + + outputs(crf_dec_l) + diff --git a/demo/semantic_role_labeling/predict.py b/demo/semantic_role_labeling/predict.py index f051d4175cf6fff43bd7f84b457ab9dd12405a15..2761814e1811e701122e0be4850526c5b290c457 100644 --- a/demo/semantic_role_labeling/predict.py +++ b/demo/semantic_role_labeling/predict.py @@ -26,7 +26,7 @@ UNK_IDX = 0 class Prediction(): - def __init__(self, train_conf, dict_file, model_dir, label_file): + def __init__(self, train_conf, dict_file, model_dir, label_file, predicate_dict_file): """ train_conf: trainer configure. dict_file: word dictionary file name. @@ -35,26 +35,41 @@ class Prediction(): self.dict = {} self.labels = {} + self.predicate_dict={} self.labels_reverse = {} - self.load_dict_label(dict_file, label_file) + self.load_dict_label(dict_file, label_file, predicate_dict_file) len_dict = len(self.dict) len_label = len(self.labels) - - conf = parse_config(train_conf, 'dict_len=' + str(len_dict) + - ',label_len=' + str(len_label) + ',is_predict=True') + len_pred = len(self.predicate_dict) + + conf = parse_config( + train_conf, + 'dict_len=' + str(len_dict) + + ',label_len=' + str(len_label) + + ',pred_len=' + str(len_pred) + + ',is_predict=True') self.network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) self.network.loadParameters(model_dir) slots = [ + integer_value_sequence(len_dict), + integer_value_sequence(len_pred), + integer_value_sequence(len_dict), + integer_value_sequence(len_dict), + integer_value_sequence(len_dict), + integer_value_sequence(len_dict), + integer_value_sequence(len_dict), + integer_value_sequence(2) + ] integer_value_sequence(len_dict), integer_value_sequence(len_dict), integer_value_sequence(len_dict), integer_value_sequence(len_dict), integer_value_sequence(len_dict), integer_value_sequence(2) ] self.converter = DataProviderConverter(slots) - def load_dict_label(self, dict_file, label_file): + def load_dict_label(self, dict_file, label_file, predicate_dict_file): """ Load dictionary from self.dict_file. """ @@ -65,39 +80,42 @@ class Prediction(): self.labels[line.strip()] = line_count self.labels_reverse[line_count] = line.strip() + for line_count, line in enumerate(open(predicate_dict_file, 'r')): + self.predicate_dict[line.strip()] = line_count def get_data(self, data_file): """ Get input data of paddle format. """ with open(data_file, 'r') as fdata: for line in fdata: - sentence, predicate, ctx_n1, ctx_0, ctx_p1, mark, label = line.strip( + sentence, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, label = line.strip( ).split('\t') words = sentence.split() sen_len = len(words) - + word_slot = [self.dict.get(w, UNK_IDX) for w in words] - predicate_slot = [self.dict.get(predicate, UNK_IDX)] * sen_len + predicate_slot = [self.predicate_dict.get(predicate, UNK_IDX)] * sen_len + ctx_n2_slot = [self.dict.get(ctx_n2, UNK_IDX)] * sen_len ctx_n1_slot = [self.dict.get(ctx_n1, UNK_IDX)] * sen_len ctx_0_slot = [self.dict.get(ctx_0, UNK_IDX)] * sen_len ctx_p1_slot = [self.dict.get(ctx_p1, UNK_IDX)] * sen_len + ctx_p2_slot = [self.dict.get(ctx_p2, UNK_IDX)] * sen_len marks = mark.split() mark_slot = [int(w) for w in marks] + + yield word_slot, predicate_slot, ctx_n2_slot, ctx_n1_slot, \ + ctx_0_slot, ctx_p1_slot, ctx_p2_slot, mark_slot - yield word_slot, predicate_slot, ctx_n1_slot, \ - ctx_0_slot, ctx_p1_slot, mark_slot - - def predict(self, data_file): + def predict(self, data_file, output_file): """ data_file: file name of input data. """ input = self.converter(self.get_data(data_file)) output = self.network.forwardTest(input) - prob = output[0]["value"] - lab = list(np.argsort(-prob)[:, 0]) + lab = output[0]["id"].tolist() - with open(data_file, 'r') as fin, open('predict.res', 'w') as fout: + with open(data_file, 'r') as fin, open(output_file, 'w') as fout: index = 0 for line in fin: sen = line.split('\t')[0] @@ -109,8 +127,8 @@ class Prediction(): def option_parser(): - usage = ("python predict.py -c config -w model_dir " - "-d word dictionary -l label_file -i input_file") + usage = ("python predict.py -c config -w model_dir " + "-d word dictionary -l label_file -i input_file -p pred_dict_file") parser = OptionParser(usage="usage: %s [options]" % usage) parser.add_option( "-c", @@ -131,6 +149,13 @@ def option_parser(): dest="label_file", default=None, help="label file") + parser.add_option( + "-p", + "--predict_dict_file", + action="store", + dest="predict_dict_file", + default=None, + help="predict_dict_file") parser.add_option( "-i", "--data", @@ -144,6 +169,14 @@ def option_parser(): dest="model_path", default=None, help="model path") + + parser.add_option( + "-o", + "--output_file", + action="store", + dest="output_file", + default=None, + help="output file") return parser.parse_args() @@ -154,10 +187,12 @@ def main(): dict_file = options.dict_file model_path = options.model_path label_file = options.label_file + predict_dict_file = options.predict_dict_file + output_file = options.output_file swig_paddle.initPaddle("--use_gpu=0") - predict = Prediction(train_conf, dict_file, model_path, label_file) - predict.predict(data_file) + predict = Prediction(train_conf, dict_file, model_path, label_file, predict_dict_file) + predict.predict(data_file,output_file) if __name__ == '__main__': diff --git a/demo/semantic_role_labeling/predict.sh b/demo/semantic_role_labeling/predict.sh index a545b9a5d591b41bdbd54905cbbffc410abc8fb0..d0acdb0bd093974485475cf796c6d41ac7899135 100644 --- a/demo/semantic_role_labeling/predict.sh +++ b/demo/semantic_role_labeling/predict.sh @@ -26,15 +26,18 @@ LOG=`get_best_pass $log` LOG=(${LOG}) best_model_path="output/pass-${LOG[1]}" - config_file=db_lstm.py -dict_file=./data/src.dict -label_file=./data/tgt.dict +dict_file=./data/wordDict.txt +label_file=./data/targetDict.txt +predicate_dict_file=./data/verbDict.txt input_file=./data/feature +output_file=predict.res python predict.py \ -c $config_file \ -w $best_model_path \ -l $label_file \ + -p $predicate_dict_file \ -d $dict_file \ - -i $input_file + -i $input_file \ + -o $output_file diff --git a/demo/semantic_role_labeling/test.sh b/demo/semantic_role_labeling/test.sh index 844649e8c0f6867dc0766e4ec6f250c5a4a004d9..c4ab44f5ca08aefd18f2851a1410aa08563925a9 100644 --- a/demo/semantic_role_labeling/test.sh +++ b/demo/semantic_role_labeling/test.sh @@ -36,4 +36,5 @@ paddle train \ --job=test \ --use_gpu=false \ --config_args=is_test=1 \ + --test_all_data_in_one_period=1 \ 2>&1 | tee 'test.log' diff --git a/demo/semantic_role_labeling/train.sh b/demo/semantic_role_labeling/train.sh index c3a22b644be0ca08a2af73a57c09657014e49bfc..420768bb2b4ebed7b135a49c5eee5e5538426ae1 100644 --- a/demo/semantic_role_labeling/train.sh +++ b/demo/semantic_role_labeling/train.sh @@ -16,11 +16,14 @@ set -e paddle train \ --config=./db_lstm.py \ + --use_gpu=0 \ + --log_period=5000 \ + --trainer_count=1 \ + --show_parameter_stats_period=5000 \ --save_dir=./output \ - --trainer_count=4 \ - --log_period=10 \ - --num_passes=500 \ - --use_gpu=false \ - --show_parameter_stats_period=10 \ + --num_passes=10000 \ + --average_test_period=10000000 \ + --init_model_path=./data \ + --load_missing_parameter_strategy=rand \ --test_all_data_in_one_period=1 \ -2>&1 | tee 'train.log' + 2>&1 | tee 'train.log' diff --git a/doc/demo/semantic_role_labeling/curve.jpg b/doc/demo/semantic_role_labeling/curve.jpg new file mode 100644 index 0000000000000000000000000000000000000000..baa35ae7f0a0b6c246f3a0d331735477ab8bcd70 Binary files /dev/null and b/doc/demo/semantic_role_labeling/curve.jpg differ diff --git a/doc/demo/semantic_role_labeling/semantic_role_labeling.md b/doc/demo/semantic_role_labeling/semantic_role_labeling.md index 890f7314582c65e9add50664006b57aa4e0709eb..e2793b2b3494160a7a80f07ec2127bd1f1a4f2e4 100644 --- a/doc/demo/semantic_role_labeling/semantic_role_labeling.md +++ b/doc/demo/semantic_role_labeling/semantic_role_labeling.md @@ -1,183 +1,200 @@ -# Semantic Role labeling Tutorial # - -Semantic role labeling (SRL) is a form of shallow semantic parsing whose goal is to discover the predicate-argument structure of each predicate in a given input sentence. SRL is useful as an intermediate step in a wide range of natural language processing tasks, such as information extraction. automatic document categorization and question answering. An instance is as following [1]: - - [ A0 He ] [ AM-MOD would ][ AM-NEG n’t ] [ V accept] [ A1 anything of value ] from [A2 those he was writing about ]. - -- V: verb -- A0: acceptor -- A1: thing accepted -- A2: accepted-from -- A3: Attribute -- AM-MOD: modal -- AM-NEG: negation - -Given the verb "accept", the chunks in sentence would play certain semantic roles. Here, the label scheme is from Penn Proposition Bank. - -To this date, most of the successful SRL systems are built on top of some form of parsing results where pre-defined feature templates over the syntactic structure are used. This tutorial will present an end-to-end system using deep bidirectional long short-term memory (DB-LSTM)[2] for solving the SRL task, which largely outperforms the previous state-of-the-art systems. The system regards SRL task as the sequence labelling problem. - -## Data Description -The relevant paper[2] takes the data set in CoNLL-2005&2012 Shared Task for training and testing. Accordingto data license, the demo adopts the test data set of CoNLL-2005, which can be reached on website. - -To download and process the original data, user just need to execute the following command: - -```bash -cd data -./get_data.sh -``` -Several new files appear in the `data `directory as follows. -```bash -conll05st-release：the test data set of CoNll-2005 shared task -test.wsj.words：the Wall Street Journal data sentences -test.wsj.props: the propositional arguments -src.dict：the dictionary of words in sentences -tgt.dict：the labels dictionary -feature: the extracted features from data set -``` - -## Training -### DB-LSTM -Please refer to the Sentiment Analysis demo to learn more about the long short-term memory unit. - -Unlike Bidirectional-LSTM that used in Sentiment Analysis demo, the DB-LSTM adopts another way to stack LSTM layer. First a standard LSTM processes the sequence in forward direction. The input and output of this LSTM layer are taken by the next LSTM layer as input, processed in reversed direction. These two standard LSTM layers compose a pair of LSTM. Then we stack LSTM layers pair after pair to obtain the deep LSTM model. - -The following figure shows a temporal expanded 2-layer DB-LSTM network. -
-![pic](./network_arch.png) -
- -### Features -Two input features play an essential role in this pipeline: predicate (pred) and argument (argu). Two other features: predicate context (ctx-p) and region mark (mr) are also adopted. Because a single predicate word can not exactly describe the predicate information, especially when the same words appear more than one times in a sentence. With the predicate context, the ambiguity can be largely eliminated. Similarly, we use region mark mr = 1 to denote the argument position if it locates in the predicate context region, or mr = 0 if does not. These four simple features are all we need for our SRL system. Features of one sample with context size set to 1 is showed as following[2]: -
-![pic](./feature.jpg) -
- -In this sample, the coresponding labelled sentence is: - -[ A1 A record date ] has [ AM-NEG n't ] been [ V set ] . - -In the demo, we adopt the feature template as above, consists of : `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` and use `B/I/O` scheme to label each argument. These features and labels are stored in `feature` file, and separated by `\t`. - -### Data Provider - -`dataprovider.py` is the python file to wrap data. `hook()` function is to define the data slots for network. The Six features and label are all IndexSlots. -``` -def hook(settings, word_dict, label_dict, **kwargs): - settings.word_dict = word_dict - settings.label_dict = label_dict - #all inputs are integral and sequential type - settings.slots = [ - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(2), - integer_value_sequence(len(label_dict))] -``` -The corresponding data iterator is as following: -``` -@provider(use_seq=True, init_hook=hook) -def process(obj, file_name): - with open(file_name, 'r') as fdata: - for line in fdata: - sentence, predicate, ctx_n1, ctx_0, ctx_p1, mark, label = line.strip().split('\t') - words = sentence.split() - sen_len = len(words) - word_slot = [obj.word_dict.get(w, UNK_IDX) for w in words] - - predicate_slot = [obj.word_dict.get(predicate, UNK_IDX)] * sen_len - ctx_n1_slot = [obj.word_dict.get(ctx_n1, UNK_IDX) ] * sen_len - ctx_0_slot = [obj.word_dict.get(ctx_0, UNK_IDX) ] * sen_len - ctx_p1_slot = [obj.word_dict.get(ctx_p1, UNK_IDX) ] * sen_len - - marks = mark.split() - mark_slot = [int(w) for w in marks] - - label_list = label.split() - label_slot = [obj.label_dict.get(w) for w in label_list] - - yield word_slot, predicate_slot, ctx_n1_slot, ctx_0_slot, ctx_p1_slot, mark_slot, label_slot -``` -The `process`function yield 7 lists which are six features and labels. - -### Neural Network Config -`db_lstm.py` is the neural network config file to load the dictionaries and define the data provider module and network architecture during the training procedure. - -Seven `data_layer` load instances from data provider. Six features are transformed into embedddings respectively, and mixed by `mixed_layer` . Deep bidirectional LSTM layers extract features for the softmax layer. The objective function is cross entropy of labels. - -### Run Training -The script for training is `train.sh`, user just need to execute: -```bash - ./train.sh -``` -The content in `train.sh`: -``` -paddle train \ - --config=./db_lstm.py \ - --save_dir=./output \ - --trainer_count=4 \ - --log_period=10 \ - --num_passes=500 \ - --use_gpu=false \ - --show_parameter_stats_period=10 \ - --test_all_data_in_one_period=1 \ -2>&1 | tee 'train.log' -``` - -- \--config=./db_lstm.py : network config file. -- \--save_di=./output: output path to save models. -- \--trainer_count=4 : set thread number (or GPU count). -- \--log_period=10 : print log every 20 batches. -- \--num_passes=500: set pass number, one pass in PaddlePaddle means training all samples in dataset one time. -- \--use_gpu=false: use CPU to train, set true, if you install GPU version of PaddlePaddle and want to use GPU to train. -- \--show_parameter_stats_period=10: show parameter statistic every 100 batches. -- \--test_all_data_in_one_period=1: test all data in every testing. - - -After training, the models will be saved in directory `output`. - -### Run testing -The script for testing is `test.sh`, user just need to execute: -```bash - ./test.sh -``` -The main part in `tesh.sh` -``` -paddle train \ - --config=./db_lstm.py \ - --model_list=$model_list \ - --job=test \ - --config_args=is_test=1 \ -``` - - - \--config=./db_lstm.py: network config file - - \--model_list=$model_list.list: model list file - - \--job=test: indicate the test job - - \--config_args=is_test=1: flag to indicate test - - -### Run prediction -The script for prediction is `predict.sh`, user just need to execute: -```bash - ./predict.sh - -``` -In `predict.sh`, user should offer the network config file, model path, label file, word dictionary file, feature file -``` -python predict.py - -c $config_file - -w $model_path - -l $label_file - -d $dict_file - -i $input_file -``` - -`predict.py` is the main executable python script, which includes functions: load model, load data, data prediction. The network model will output the probability distribution of labels. In the demo, we take the label with maximum probability as result. User can also implement the beam search or viterbi decoding upon the probability distribution matrix. - -After prediction, the result is saved in `predict.res`. - -## Reference -[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005. - -[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015. +# Semantic Role labeling Tutorial # + +Semantic role labeling (SRL) is a form of shallow semantic parsing whose goal is to discover the predicate-argument structure of each predicate in a given input sentence. SRL is useful as an intermediate step in a wide range of natural language processing tasks, such as information extraction. automatic document categorization and question answering. An instance is as following [1]: + + [ A0 He ] [ AM-MOD would ][ AM-NEG n’t ] [ V accept] [ A1 anything of value ] from [A2 those he was writing about ]. + +- V: verb +- A0: acceptor +- A1: thing accepted +- A2: accepted-from +- A3: Attribute +- AM-MOD: modal +- AM-NEG: negation + +Given the verb "accept", the chunks in sentence would play certain semantic roles. Here, the label scheme is from Penn Proposition Bank. + +To this date, most of the successful SRL systems are built on top of some form of parsing results where pre-defined feature templates over the syntactic structure are used. This tutorial will present an end-to-end system using deep bidirectional long short-term memory (DB-LSTM)[2] for solving the SRL task, which largely outperforms the previous state-of-the-art systems. The system regards SRL task as the sequence labelling problem. + +## Data Description +The relevant paper[2] takes the data set in CoNLL-2005&2012 Shared Task for training and testing. Accordingto data license, the demo adopts the test data set of CoNLL-2005, which can be reached on website. + +To download and process the original data, user just need to execute the following command: + +```bash +cd data +./get_data.sh +``` +Several new files appear in the `data `directory as follows. +```bash +conll05st-release：the test data set of CoNll-2005 shared task +test.wsj.words：the Wall Street Journal data sentences +test.wsj.props: the propositional arguments +feature: the extracted features from data set +``` + +## Training +### DB-LSTM +Please refer to the Sentiment Analysis demo to learn more about the long short-term memory unit. + +Unlike Bidirectional-LSTM that used in Sentiment Analysis demo, the DB-LSTM adopts another way to stack LSTM layer. First a standard LSTM processes the sequence in forward direction. The input and output of this LSTM layer are taken by the next LSTM layer as input, processed in reversed direction. These two standard LSTM layers compose a pair of LSTM. Then we stack LSTM layers pair after pair to obtain the deep LSTM model. + +The following figure shows a temporal expanded 2-layer DB-LSTM network. +
+![pic](./network_arch.png) +
+ +### Features +Two input features play an essential role in this pipeline: predicate (pred) and argument (argu). Two other features: predicate context (ctx-p) and region mark (mr) are also adopted. Because a single predicate word can not exactly describe the predicate information, especially when the same words appear more than one times in a sentence. With the predicate context, the ambiguity can be largely eliminated. Similarly, we use region mark mr = 1 to denote the argument position if it locates in the predicate context region, or mr = 0 if does not. These four simple features are all we need for our SRL system. Features of one sample with context size set to 1 is showed as following[2]: +
+![pic](./feature.jpg) +
+ +In this sample, the coresponding labelled sentence is: + +[ A1 A record date ] has [ AM-NEG n't ] been [ V set ] . + +In the demo, we adopt the feature template as above, consists of : `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` and use `B/I/O` scheme to label each argument. These features and labels are stored in `feature` file, and separated by `\t`. + +### Data Provider + +`dataprovider.py` is the python file to wrap data. `hook()` function is to define the data slots for network. The Six features and label are all IndexSlots. +``` +def hook(settings, word_dict, label_dict, **kwargs): + settings.word_dict = word_dict + settings.label_dict = label_dict + #all inputs are integral and sequential type + settings.slots = [ + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(predicate_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(2), + integer_value_sequence(len(label_dict))] +``` +The corresponding data iterator is as following: +``` +@provider(init_hook=hook, should_shuffle=True, calc_batch_size=get_batch_size, + can_over_batch_size=False, cache=CacheType.CACHE_PASS_IN_MEM) +def process(settings, file_name): + with open(file_name, 'r') as fdata: + for line in fdata: + sentence, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, label = \ + line.strip().split('\t') + + words = sentence.split() + sen_len = len(words) + word_slot = [settings.word_dict.get(w, UNK_IDX) for w in words] + + predicate_slot = [settings.predicate_dict.get(predicate)] * sen_len + ctx_n2_slot = [settings.word_dict.get(ctx_n2, UNK_IDX)] * sen_len + ctx_n1_slot = [settings.word_dict.get(ctx_n1, UNK_IDX)] * sen_len + ctx_0_slot = [settings.word_dict.get(ctx_0, UNK_IDX)] * sen_len + ctx_p1_slot = [settings.word_dict.get(ctx_p1, UNK_IDX)] * sen_len + ctx_p2_slot = [settings.word_dict.get(ctx_p2, UNK_IDX)] * sen_len + + marks = mark.split() + mark_slot = [int(w) for w in marks] + + label_list = label.split() + label_slot = [settings.label_dict.get(w) for w in label_list] + yield word_slot, predicate_slot, ctx_n2_slot, ctx_n1_slot, \ + ctx_0_slot, ctx_p1_slot, ctx_p2_slot, mark_slot, label_slot +``` +The `process`function yield 9 lists which are 8 features and label. + +### Neural Network Config +`db_lstm.py` is the neural network config file to load the dictionaries and define the data provider module and network architecture during the training procedure. + +Nine `data_layer` load instances from data provider. Eight features are transformed into embedddings respectively, and mixed by `mixed_layer` . Deep bidirectional LSTM layers extract features for the softmax layer. The objective function is cross entropy of labels. + +### Run Training +The script for training is `train.sh`, user just need to execute: +```bash + ./train.sh +``` +The content in `train.sh`: +``` +paddle train \ + --config=./db_lstm.py \ + --use_gpu=0 \ + --log_period=5000 \ + --trainer_count=1 \ + --show_parameter_stats_period=5000 \ + --save_dir=./output \ + --num_passes=10000 \ + --average_test_period=10000000 \ + --init_model_path=./data \ + --load_missing_parameter_strategy=rand \ + --test_all_data_in_one_period=1 \ +2>&1 | tee 'train.log' +``` + +- \--config=./db_lstm.py : network config file. +- \--use_gpu=false: use CPU to train, set true, if you install GPU version of PaddlePaddle and want to use GPU to train, until now crf_layer do not support GPU +- \--log_period=500: print log every 20 batches. +- \--trainer_count=1: set thread number (or GPU count). +- \--show_parameter_stats_period=5000: show parameter statistic every 100 batches. +- \--save_dir=./output: output path to save models. +- \--num_passes=10000: set pass number, one pass in PaddlePaddle means training all samples in dataset one time. +- \--average_test_period=10000000: do test on average parameter every average_test_period batches +- \--init_model_path=./data: parameter initialization path +- \--load_missing_parameter_strategy=rand: random initialization unexisted parameters +- \--test_all_data_in_one_period=1: test all data in one period + + +After training, the models will be saved in directory `output`. Our training curve is as following: +
+![pic](./curve.jpg) +
+ +### Run testing +The script for testing is `test.sh`, user just need to execute: +```bash + ./test.sh +``` +The main part in `tesh.sh` +``` +paddle train \ + --config=./db_lstm.py \ + --model_list=$model_list \ + --job=test \ + --config_args=is_test=1 \ +``` + + - \--config=./db_lstm.py: network config file + - \--model_list=$model_list.list: model list file + - \--job=test: indicate the test job + - \--config_args=is_test=1: flag to indicate test + - \--test_all_data_in_one_period=1: test all data in 1 period + + +### Run prediction +The script for prediction is `predict.sh`, user just need to execute: +```bash + ./predict.sh + +``` +In `predict.sh`, user should offer the network config file, model path, label file, word dictionary file, feature file +``` +python predict.py + -c $config_file \ + -w $best_model_path \ + -l $label_file \ + -p $predicate_dict_file \ + -d $dict_file \ + -i $input_file \ + -o $output_file +``` + +`predict.py` is the main executable python script, which includes functions: load model, load data, data prediction. The network model will output the probability distribution of labels. In the demo, we take the label with maximum probability as result. User can also implement the beam search or viterbi decoding upon the probability distribution matrix. + +After prediction, the result is saved in `predict.res`. + +## Reference +[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005. + +[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015. diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt index cdb730bb3cec7a32fa42cf4c6738d575b76c6032..11dbfb54b268774405ade1e532bef9a0e8c7ada9 100755 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -81,5 +81,8 @@ else() add_library(paddle_cuda ${CUDA_SOURCES}) endif() -add_style_check_target(paddle_cuda ${CUDA_SOURCES}) -add_style_check_target(paddle_cuda ${CUDA_HEADERS}) +add_style_check_target(paddle_cuda + ${CUDA_SOURCES} + ${CUDA_HEADERS} + ${CUDA_DSO_SOURCES} + ${CUDA_CXX_WITH_GPU_SOURCES}) diff --git a/paddle/cuda/src/hl_cuda_cublas.cc b/paddle/cuda/src/hl_cuda_cublas.cc index f16376ec937d3a397d9e7117de528c304f8403ee..abf6afadc218f615dc6b3cf734d09f072214be40 100644 --- a/paddle/cuda/src/hl_cuda_cublas.cc +++ b/paddle/cuda/src/hl_cuda_cublas.cc @@ -104,7 +104,7 @@ CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) #endif const char* hl_cublas_get_error_string(cublasStatus_t status) { - switch(status) { + switch (status) { case CUBLAS_STATUS_NOT_INITIALIZED: return "[cublas status]: not initialized"; case CUBLAS_STATUS_ALLOC_FAILED: @@ -181,7 +181,7 @@ void hl_matrix_inverse(real *A_d, real *C_d, int dimN, int lda, int ldc) { real **inout_d = (real **)hl_malloc_device(sizeof(real *)); hl_memcpy(inout_d, inout_h, sizeof(real *)); - int *pivot_d = (int *)hl_malloc_device(dimN*sizeof(int)); + int *pivot_d = (int *)hl_malloc_device(dimN * sizeof(int)); int *info_d = (int *)t_resource.gpu_mem; /* Note: cublasSgetrfBatched is used to calculate a number of @@ -189,10 +189,9 @@ void hl_matrix_inverse(real *A_d, real *C_d, int dimN, int lda, int ldc) { the API for better performance. */ CHECK_CUBLAS(CUBLAS_GETRF(t_resource.handle, - dimN, inout_d, lda, pivot_d, - info_d, 1)); + dimN, inout_d, lda, pivot_d, info_d, 1)); - int info_h; + int info_h; hl_memcpy(&info_h, info_d, sizeof(int)); if (info_h != 0) { LOG(FATAL) << "Factorization of matrix failed: matrix may be singular.\n"; @@ -204,8 +203,8 @@ void hl_matrix_inverse(real *A_d, real *C_d, int dimN, int lda, int ldc) { hl_memcpy(out_d, out_h, sizeof(real *)); CHECK_CUBLAS(CUBLAS_GETRI(t_resource.handle, - dimN, (const real **)inout_d, lda, pivot_d, - out_d, ldc, info_d, 1)); + dimN, (const real **)inout_d, lda, pivot_d, + out_d, ldc, info_d, 1)); hl_memcpy(&info_h, info_d, sizeof(int)); if (info_h != 0) { @@ -215,7 +214,7 @@ void hl_matrix_inverse(real *A_d, real *C_d, int dimN, int lda, int ldc) { hl_free_mem_device(inout_d); hl_free_mem_device(pivot_d); hl_free_mem_device(out_d); - + CHECK_SYNC("hl_matrix_inverse failed"); } diff --git a/paddle/cuda/src/hl_cuda_cudnn.cc b/paddle/cuda/src/hl_cuda_cudnn.cc index 92b28e4345c3d4d306e6ee2a7f9f50189454f951..1829fe23ac594e63253df23b350b16cb28eaebc1 100644 --- a/paddle/cuda/src/hl_cuda_cudnn.cc +++ b/paddle/cuda/src/hl_cuda_cudnn.cc @@ -159,13 +159,11 @@ CUDNN_DNN_ROUTINE_EACH_R5(DYNAMIC_LOAD_CUDNN_WRAP) bool g_is_libcudnn_init = false; int g_cudnn_lib_version = 0; -void hl_cudnn_desc_init(cudnnTensorDescriptor_t* cudnn_desc) -{ +void hl_cudnn_desc_init(cudnnTensorDescriptor_t* cudnn_desc) { CHECK_CUDNN(dynload::cudnnCreateTensorDescriptor(cudnn_desc)); } -void hl_cudnn_init(cudnnHandle_t *cudnn_handle, cudaStream_t stream) -{ +void hl_cudnn_init(cudnnHandle_t *cudnn_handle, cudaStream_t stream) { size_t cudnn_dso_ver = dynload::cudnnGetVersion(); size_t cudnn_dso_major = cudnn_dso_ver / 1000; size_t cudnn_cuh_major = CUDNN_VERSION / 1000; @@ -212,13 +210,18 @@ void hl_conv_workspace(hl_tensor_descriptor input, CHECK_NOTNULL(conv); // Specify workspace limit directly - size_t memoryLimitBytes = (1LL << 20) * FLAGS_cudnn_conv_workspace_limit_in_mb; + size_t memoryLimitBytes = + (1LL << 20) * FLAGS_cudnn_conv_workspace_limit_in_mb; // cudnn convolution forward configuration - cudnnTensorDescriptor_t fwd_src_desc = GET_TENSOR_DESCRIPTOR(input); - cudnnTensorDescriptor_t fwd_dest_desc = GET_TENSOR_DESCRIPTOR(output); - cudnnFilterDescriptor_t fwd_filter_desc = GET_FILTER_DESCRIPTOR(filter); - cudnnConvolutionDescriptor_t fwd_conv_desc = GET_CONVOLUTION_DESCRIPTOR(conv); + cudnnTensorDescriptor_t fwd_src_desc = + GET_TENSOR_DESCRIPTOR(input); + cudnnTensorDescriptor_t fwd_dest_desc = + GET_TENSOR_DESCRIPTOR(output); + cudnnFilterDescriptor_t fwd_filter_desc = + GET_FILTER_DESCRIPTOR(filter); + cudnnConvolutionDescriptor_t fwd_conv_desc = + GET_CONVOLUTION_DESCRIPTOR(conv); CHECK_CUDNN(dynload::cudnnGetConvolutionForwardAlgorithm( t_resource.cudnn_handle, @@ -250,23 +253,23 @@ void hl_conv_workspace(hl_tensor_descriptor input, GET_CONVOLUTION_DESCRIPTOR(conv); CHECK_CUDNN(dynload::cudnnGetConvolutionBackwardDataAlgorithm( - t_resource.cudnn_handle, - bwd_data_filter_desc, - bwd_data_diff_desc, - bwd_data_conv_desc, - bwd_data_grad_desc, - CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, - memoryLimitBytes, - reinterpret_cast(convBwdDataAlgo))); + t_resource.cudnn_handle, + bwd_data_filter_desc, + bwd_data_diff_desc, + bwd_data_conv_desc, + bwd_data_grad_desc, + CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT, + memoryLimitBytes, + reinterpret_cast(convBwdDataAlgo))); CHECK_CUDNN(dynload::cudnnGetConvolutionBackwardDataWorkspaceSize( - t_resource.cudnn_handle, - bwd_data_filter_desc, - bwd_data_diff_desc, - bwd_data_conv_desc, - bwd_data_grad_desc, - static_cast(*convBwdDataAlgo), - bwdDataLimitBytes)); + t_resource.cudnn_handle, + bwd_data_filter_desc, + bwd_data_diff_desc, + bwd_data_conv_desc, + bwd_data_grad_desc, + static_cast(*convBwdDataAlgo), + bwdDataLimitBytes)); // cudnn convolution backward filter configuration cudnnTensorDescriptor_t bwd_filter_src_desc = @@ -279,21 +282,21 @@ void hl_conv_workspace(hl_tensor_descriptor input, GET_FILTER_DESCRIPTOR(filter); CHECK_CUDNN(dynload::cudnnGetConvolutionBackwardFilterAlgorithm( - t_resource.cudnn_handle, - bwd_filter_src_desc, - bwd_filter_diff_desc, - bwd_filter_conv_desc, - bwd_filter_grad_desc, - CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, - memoryLimitBytes, - reinterpret_cast(convBwdFilterAlgo))); + t_resource.cudnn_handle, + bwd_filter_src_desc, + bwd_filter_diff_desc, + bwd_filter_conv_desc, + bwd_filter_grad_desc, + CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT, + memoryLimitBytes, + reinterpret_cast(convBwdFilterAlgo))); CHECK_CUDNN(dynload::cudnnGetConvolutionBackwardFilterWorkspaceSize( - t_resource.cudnn_handle, bwd_filter_src_desc, - bwd_filter_diff_desc, bwd_filter_conv_desc, - bwd_filter_grad_desc, - static_cast(*convBwdFilterAlgo), - bwdFilterLimitBytes)); + t_resource.cudnn_handle, bwd_filter_src_desc, + bwd_filter_diff_desc, bwd_filter_conv_desc, + bwd_filter_grad_desc, + static_cast(*convBwdFilterAlgo), + bwdFilterLimitBytes)); #endif } @@ -302,8 +305,7 @@ void hl_create_tensor_descriptor(hl_tensor_descriptor* image_desc, int batch_size, int feature_maps, int height, - int width) -{ + int width) { CHECK_NOTNULL(image_desc); cudnn_tensor_descriptor hl_desc = @@ -359,8 +361,7 @@ void hl_tensor_reshape(hl_tensor_descriptor image_desc, int batch_size, int feature_maps, int height, - int width) -{ + int width) { const int stride_w = 1; const int stride_h = width * stride_w; const int stride_c = height * stride_h; @@ -384,8 +385,7 @@ void hl_tensor_reshape(hl_tensor_descriptor image_desc, int nStride, int cStride, int hStride, - int wStride) -{ + int wStride) { CHECK_NOTNULL(image_desc); cudnn_tensor_descriptor hl_desc = (cudnn_tensor_descriptor)image_desc; @@ -408,8 +408,7 @@ void hl_tensor_reshape(hl_tensor_descriptor image_desc, hl_desc->width = width; } -void hl_destroy_tensor_descriptor(hl_tensor_descriptor image_desc) -{ +void hl_destroy_tensor_descriptor(hl_tensor_descriptor image_desc) { CHECK_NOTNULL(image_desc); cudnn_tensor_descriptor hl_desc = (cudnn_tensor_descriptor)image_desc; @@ -430,11 +429,9 @@ void hl_create_pooling_descriptor(hl_pooling_descriptor* pooling_desc, int height_padding, int width_padding, int stride_height, - int stride_width) -{ + int stride_width) { cudnnPoolingMode_t cudnn_mode; - switch (mode) - { + switch (mode) { case HL_POOLING_MAX: cudnn_mode = CUDNN_POOLING_MAX; break; @@ -478,13 +475,13 @@ void hl_create_pooling_descriptor(hl_pooling_descriptor* pooling_desc, *pooling_desc = (hl_pooling_descriptor)hl_pooling_desc; } -void hl_destroy_pooling_descriptor(hl_pooling_descriptor pooling_desc) -{ +void hl_destroy_pooling_descriptor(hl_pooling_descriptor pooling_desc) { CHECK_NOTNULL(pooling_desc); - cudnn_pooling_descriptor hl_pooling = (cudnn_pooling_descriptor)pooling_desc; - CHECK_NOTNULL(hl_pooling->desc); + cudnn_pooling_descriptor hl_pooling = + (cudnn_pooling_descriptor)pooling_desc; + CHECK_NOTNULL(hl_pooling->desc); CHECK_CUDNN(dynload::cudnnDestroyPoolingDescriptor(hl_pooling->desc)); hl_pooling->desc = NULL; @@ -496,8 +493,7 @@ void hl_pooling_forward(hl_tensor_descriptor input, real* input_image, hl_tensor_descriptor output, real* output_image, - hl_pooling_descriptor pooling) -{ + hl_pooling_descriptor pooling) { cudnnPoolingDescriptor_t pooling_desc; cudnnTensorDescriptor_t input_desc; cudnnTensorDescriptor_t output_desc; @@ -531,8 +527,7 @@ void hl_pooling_backward(hl_tensor_descriptor input, hl_tensor_descriptor output, real* output_image, real* output_image_grad, - hl_pooling_descriptor pooling) -{ + hl_pooling_descriptor pooling) { cudnnPoolingDescriptor_t pooling_desc; cudnnTensorDescriptor_t input_desc; cudnnTensorDescriptor_t output_desc; @@ -571,8 +566,7 @@ void hl_create_filter_descriptor(hl_filter_descriptor* filter, int input_feature_maps, int output_feature_maps, int height, - int width) -{ + int width) { CHECK_NOTNULL(filter); cudnn_filter_descriptor hl_filter = @@ -607,8 +601,7 @@ void hl_create_filter_descriptor(hl_filter_descriptor* filter, } -void hl_destroy_filter_descriptor(hl_filter_descriptor filter) -{ +void hl_destroy_filter_descriptor(hl_filter_descriptor filter) { CHECK_NOTNULL(filter); cudnn_filter_descriptor hl_filter = (cudnn_filter_descriptor)filter; @@ -627,14 +620,13 @@ void hl_create_convolution_descriptor(hl_convolution_descriptor* conv, int padding_height, int padding_width, int stride_height, - int stride_width) -{ + int stride_width) { CHECK_NOTNULL(conv); - cudnn_convolution_descriptor hl_conv = - (cudnn_convolution_descriptor)malloc(sizeof(_cudnn_convolution_descriptor)); - CHECK_NOTNULL(hl_conv); + cudnn_convolution_descriptor hl_conv = (cudnn_convolution_descriptor) + malloc(sizeof(_cudnn_convolution_descriptor)); + CHECK_NOTNULL(hl_conv); CHECK_CUDNN(dynload::cudnnCreateConvolutionDescriptor(&hl_conv->desc)); cudnnConvolutionMode_t mode = CUDNN_CROSS_CORRELATION; @@ -667,8 +659,7 @@ void hl_reset_convolution_descriptor(hl_convolution_descriptor conv, int padding_height, int padding_width, int stride_height, - int stride_width) -{ + int stride_width) { CHECK_NOTNULL(conv); CHECK_NOTNULL(image); CHECK_NOTNULL(filter); @@ -697,8 +688,7 @@ void hl_reset_convolution_descriptor(hl_convolution_descriptor conv, hl_conv->mode = mode; } -void hl_destroy_convolution_descriptor(hl_convolution_descriptor conv) -{ +void hl_destroy_convolution_descriptor(hl_convolution_descriptor conv) { CHECK_NOTNULL(conv); cudnn_convolution_descriptor hl_conv = (cudnn_convolution_descriptor)conv; @@ -753,8 +743,7 @@ void hl_convolution_forward(hl_tensor_descriptor input, void hl_convolution_forward_add_bias(hl_tensor_descriptor bias, real* bias_data, hl_tensor_descriptor output, - real* output_data) -{ + real* output_data) { CHECK_NOTNULL(bias); CHECK_NOTNULL(output); CHECK_NOTNULL(bias_data); @@ -782,8 +771,7 @@ void hl_convolution_forward_add_bias(hl_tensor_descriptor bias, void hl_convolution_backward_bias(hl_tensor_descriptor bias, real* bias_grad_data, hl_tensor_descriptor output, - real* output_grad_data) -{ + real* output_grad_data) { CHECK_NOTNULL(bias); CHECK_NOTNULL(output); CHECK_NOTNULL(bias_grad_data); @@ -814,7 +802,6 @@ void hl_convolution_backward_filter(hl_tensor_descriptor input, void* gpuWorkSpace, size_t sizeInBytes, int convBwdFilterAlgo) { - CHECK_NOTNULL(input); CHECK_NOTNULL(output); CHECK_NOTNULL(filter); @@ -889,8 +876,7 @@ void hl_convolution_backward_data(hl_tensor_descriptor input, void hl_softmax_forward(real *input, real *output, int height, - int width) -{ + int width) { #ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else @@ -923,8 +909,7 @@ void hl_softmax_forward(real *input, void hl_softmax_backward(real *output_value, real *output_grad, int height, - int width) -{ + int width) { #ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else diff --git a/paddle/cuda/src/hl_cuda_device.cc b/paddle/cuda/src/hl_cuda_device.cc index 3ea2c91bd5a41e0cd6ece0605a25e645676faa40..ca19f210c5c9d5151b01ce81a4f44663e2df97cc 100644 --- a/paddle/cuda/src/hl_cuda_device.cc +++ b/paddle/cuda/src/hl_cuda_device.cc @@ -203,8 +203,8 @@ inline pid_t gettid() { #endif pid_t tid = syscall(__NR_gettid); #endif - CHECK_NE(tid, -1); - return tid; + CHECK_NE((int)tid, -1); + return tid; } void hl_init(int device) { @@ -355,7 +355,8 @@ void* hl_malloc_host(size_t size) { void *dest_h; CHECK(size) << __func__ << ": the size for device memory is 0, please check."; - CHECK_CUDA(dynload::cudaHostAlloc((void**)&dest_h, size, cudaHostAllocDefault)); + CHECK_CUDA(dynload::cudaHostAlloc( + (void**)&dest_h, size, cudaHostAllocDefault)); return dest_h; } @@ -364,7 +365,7 @@ void hl_free_mem_host(void *dest_h) { CHECK_NOTNULL(dest_h); cudaError_t err = dynload::cudaFreeHost(dest_h); - CHECK (cudaSuccess == err || cudaErrorCudartUnloading == err) + CHECK(cudaSuccess == err || cudaErrorCudartUnloading == err) << hl_get_device_error_string(); } @@ -502,7 +503,8 @@ int hl_get_cuda_version() { return g_cuda_lib_version; } -void hl_create_thread_resources(int device, thread_device_resources device_res) { +void hl_create_thread_resources(int device, + thread_device_resources device_res) { CHECK_CUDA(dynload::cudaSetDevice(device)); /* create thread stream */ diff --git a/paddle/cuda/src/hl_cudart_wrap.cc b/paddle/cuda/src/hl_cudart_wrap.cc index 27bbd03bc328293d978867c6badddc13a754ece2..fe755b8c2606dffeeff2ea1549180ca8b134c251 100644 --- a/paddle/cuda/src/hl_cudart_wrap.cc +++ b/paddle/cuda/src/hl_cudart_wrap.cc @@ -78,48 +78,38 @@ __host__ cudaError_t CUDARTAPI cudaLaunchKernel(const void *func, dim3 blockDim, void **args, size_t sharedMem, - cudaStream_t stream) -{ - return dynload::cudaLaunchKernel(func, gridDim, blockDim, args, sharedMem, stream); + cudaStream_t stream) { + return dynload::cudaLaunchKernel(func, gridDim, blockDim, + args, sharedMem, stream); } #endif /* CUDART_VERSION >= 7000 */ -__host__ cudaError_t CUDARTAPI cudaLaunch(const void *func) -{ +__host__ cudaError_t CUDARTAPI cudaLaunch(const void *func) { return dynload::cudaLaunch(func); } __host__ cudaError_t CUDARTAPI cudaSetupArgument(const void *arg, size_t size, - size_t offset) -{ + size_t offset) { return dynload::cudaSetupArgument(arg, size, offset); } __host__ cudaError_t CUDARTAPI cudaConfigureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, - cudaStream_t stream) -{ + cudaStream_t stream) { return dynload::cudaConfigureCall(gridDim, blockDim, sharedMem, stream); } extern "C" { -void** CUDARTAPI __cudaRegisterFatBinary( - void *fatCubin -) -{ +void** CUDARTAPI __cudaRegisterFatBinary(void *fatCubin) { return dynload::__cudaRegisterFatBinary(fatCubin); - } -void CUDARTAPI __cudaUnregisterFatBinary( - void **fatCubinHandle -) -{ +void CUDARTAPI __cudaUnregisterFatBinary(void **fatCubinHandle) { return dynload::__cudaUnregisterFatBinary(fatCubinHandle); } diff --git a/paddle/cuda/src/hl_dso_loader.cc b/paddle/cuda/src/hl_dso_loader.cc index b564b969033680a001577de25ceb84dae391754a..5cb16cfbb372209a6cac83cdaace9afbf590e0fe 100644 --- a/paddle/cuda/src/hl_dso_loader.cc +++ b/paddle/cuda/src/hl_dso_loader.cc @@ -19,17 +19,18 @@ limitations under the License. */ P_DEFINE_string(cudnn_dir, "", "Specify path for loading libcudnn.so. For instance, " - "/usr/local/cudnn/lib64. If empty [default], dlopen will search " - "cudnn from LD_LIBRARY_PATH"); + "/usr/local/cudnn/lib64. If empty [default], dlopen " + "will search cudnn from LD_LIBRARY_PATH"); P_DEFINE_string(cuda_dir, "", "Specify path for loading cuda library, such as libcublas, " - "libcurand. For instance, /usr/local/cuda/lib64. " - "(Note: libcudart can not be specified by cuda_dir, since some " + "libcurand. For instance, /usr/local/cuda/lib64. (Note: " + "libcudart can not be specified by cuda_dir, since some " "build-in function in cudart already ran before main entry). " - "If empty [default], dlopen will search cuda from LD_LIBRARY_PATH"); + "If default, dlopen will search cuda from LD_LIBRARY_PATH"); -static inline std::string join(const std::string& part1, const std::string& part2) { +static inline std::string join(const std::string& part1, + const std::string& part2) { // directory separator const char sep = '/'; @@ -49,10 +50,10 @@ static inline std::string join(const std::string& part1, const std::string& part static inline void GetDsoHandleFromDefaultPath( std::string& dso_path, void** dso_handle, int dynload_flags) { VLOG(3) << "Try to find cuda library: " << dso_path - << " from default system path."; - // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH + << " from default system path."; + // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH *dso_handle = dlopen(dso_path.c_str(), dynload_flags); - + // DYLD_LIBRARY_PATH is disabled after Mac OS 10.11 to // bring System Integrity Projection (SIP), if dso_handle // is null, search from default package path in Mac OS. @@ -62,13 +63,13 @@ static inline void GetDsoHandleFromDefaultPath( *dso_handle = dlopen(dso_path.c_str(), dynload_flags); if (nullptr == *dso_handle) { if (dso_path == "libcudnn.dylib") { - LOG(FATAL) << "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n" - << "For instance, sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C " - << "/usr/local \n sudo chmod a+r /usr/local/cuda/include/cudnn.h " + LOG(FATAL) << "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n" // NOLINT + << "For instance, sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C " // NOLINT + << "/usr/local \n sudo chmod a+r /usr/local/cuda/include/cudnn.h " // NOLINT << "/usr/local/cuda/lib/libcudnn*"; } - } - } + } + } #endif } @@ -96,19 +97,19 @@ static inline void GetDsoHandleFromSearchPath( CHECK(nullptr != *dso_handle) << "Failed to find cuda library: " << dlPath << std::endl - << "Please specify its path correctly using one of the following ideas: \n" + << "Please specify its path correctly using one of the following ways: \n" // NOLINT - << "Idea 1. set cuda and cudnn lib path at runtime. " - << "http://www.paddlepaddle.org/doc/ui/cmd_argument/argument_outline.html \n" + << "Method 1. set cuda and cudnn lib path at runtime. " + << "http://www.paddlepaddle.org/doc/ui/cmd_argument/argument_outline.html \n" // NOLINT << "For instance, issue command: paddle train --use_gpu=1 " - << "--cuda_dir=/usr/local/cudnn/lib --cudnn_dir=/usr/local/cudnn/lib ...\n" + << "--cuda_dir=/usr/local/cuda/lib64 --cudnn_dir=/usr/local/cudnn/lib ...\n" // NOLINT - << "Idea 2. set environment variable LD_LIBRARY_PATH on Linux or " + << "Method 2. set environment variable LD_LIBRARY_PATH on Linux or " << "DYLD_LIBRARY_PATH on Mac OS. \n" << "For instance, issue command: export LD_LIBRARY_PATH=... \n" << "Note: After Mac OS 10.11, using the DYLD_LIBRARY_PATH is impossible " - << "unless System Integrity Protection (SIP) is disabled. However, @Idea 1" + << "unless System Integrity Protection (SIP) is disabled. However, method 1 " // NOLINT << "always work well."; }