From 3ceffb4af04bc106cdea8592840c36e0dfff5e16 Mon Sep 17 00:00:00 2001 From: Steffy-zxf <48793257+Steffy-zxf@users.noreply.github.com> Date: Tue, 11 Aug 2020 10:53:11 +0800 Subject: [PATCH] Update matching task (#814) --- .../ernie_pairwise_matching.py | 6 +- .../ernie_pairwise_matching_predict.py | 8 +- .../run_embedding_pairwise_matching.sh | 4 +- ...run_embedding_pairwise_matching_predict.sh | 2 +- .../embedding_pointwise_matching_predict.py | 8 +- .../ernie_pointwise_matching.py | 4 +- .../ernie_pointwise_matching_predict.py | 14 +- .../run_embedding_pointwise_matching.sh | 2 +- paddlehub/finetune/task/matching_task.py | 138 ++++++++++++++++-- paddlehub/version.py | 2 +- 10 files changed, 153 insertions(+), 35 deletions(-) diff --git a/demo/pairwise_text_matching/ernie_pairwise_matching.py b/demo/pairwise_text_matching/ernie_pairwise_matching.py index 70a2e4a1..a7899c01 100644 --- a/demo/pairwise_text_matching/ernie_pairwise_matching.py +++ b/demo/pairwise_text_matching/ernie_pairwise_matching.py @@ -52,9 +52,9 @@ if __name__ == '__main__': # Construct transfer learning network # Use sequence-level output. - query = outputs["pooled_output"] - left = outputs['pooled_output_2'] - right = outputs['pooled_output_3'] + query = outputs["sequence_output"] + left = outputs['sequence_output_2'] + right = outputs['sequence_output_3'] # Select fine-tune strategy strategy = hub.AdamWeightDecayStrategy( diff --git a/demo/pairwise_text_matching/ernie_pairwise_matching_predict.py b/demo/pairwise_text_matching/ernie_pairwise_matching_predict.py index 128edf48..c26a0642 100644 --- a/demo/pairwise_text_matching/ernie_pairwise_matching_predict.py +++ b/demo/pairwise_text_matching/ernie_pairwise_matching_predict.py @@ -48,9 +48,9 @@ if __name__ == '__main__': # Construct transfer learning network. # Use sequence-level output. - query = outputs["pooled_output"] - left = outputs['pooled_output_2'] - right = outputs['pooled_output_3'] + query = outputs["sequence_output"] + left = outputs['sequence_output_2'] + right = outputs['sequence_output_3'] # Select fine-tune strategy. strategy = hub.AdamWeightDecayStrategy() @@ -91,6 +91,6 @@ if __name__ == '__main__': max_seq_len=args.max_seq_len, label_list=dataset.get_labels(), return_result=True, - accelerate_mode=False) + accelerate_mode=True) for index, text in enumerate(text_pairs): print("data: %s, prediction_label: %s" % (text, results[index])) diff --git a/demo/pairwise_text_matching/run_embedding_pairwise_matching.sh b/demo/pairwise_text_matching/run_embedding_pairwise_matching.sh index f5674c69..9f61059a 100644 --- a/demo/pairwise_text_matching/run_embedding_pairwise_matching.sh +++ b/demo/pairwise_text_matching/run_embedding_pairwise_matching.sh @@ -3,7 +3,7 @@ CKPT_DIR="./ckpt_embedding_pairwise_matching" python -u embedding_pairwise_matching.py \ --batch_size=128 \ --checkpoint_dir=${CKPT_DIR} \ - --learning_rate=5e-2 \ + --learning_rate=5e-3 \ --max_seq_len=128 \ - --num_epoch=300 \ + --num_epoch=100 \ --network=bow diff --git a/demo/pairwise_text_matching/run_embedding_pairwise_matching_predict.sh b/demo/pairwise_text_matching/run_embedding_pairwise_matching_predict.sh index 8d3e9c07..88fe45b1 100644 --- a/demo/pairwise_text_matching/run_embedding_pairwise_matching_predict.sh +++ b/demo/pairwise_text_matching/run_embedding_pairwise_matching_predict.sh @@ -4,4 +4,4 @@ python -u embedding_pairwise_matching_predict.py \ --batch_size=1 \ --checkpoint_dir=${CKPT_DIR} \ --max_seq_len=128 \ - --network=lstm + --network=bow diff --git a/demo/pointwise_text_matching/embedding_pointwise_matching_predict.py b/demo/pointwise_text_matching/embedding_pointwise_matching_predict.py index da5ed8ae..204fa25d 100644 --- a/demo/pointwise_text_matching/embedding_pointwise_matching_predict.py +++ b/demo/pointwise_text_matching/embedding_pointwise_matching_predict.py @@ -88,12 +88,12 @@ if __name__ == '__main__': # Prediction data sample. text_pairs = [ [ - "请问不是您的账户吗？", # query - "您好，请问您使用的邮箱类型是？" # title + "淘宝上怎么用信用卡分期付款", # query + "淘宝上怎么分期付款，没有信用卡", # title ], [ - "推荐个手机游戏", # query - "手机游戏推荐" # title + "山楂干怎么吃好吃？", # query + "山楂怎么做好吃", # title ] ] diff --git a/demo/pointwise_text_matching/ernie_pointwise_matching.py b/demo/pointwise_text_matching/ernie_pointwise_matching.py index 337f6dd1..383b8060 100644 --- a/demo/pointwise_text_matching/ernie_pointwise_matching.py +++ b/demo/pointwise_text_matching/ernie_pointwise_matching.py @@ -52,8 +52,8 @@ if __name__ == '__main__': # Construct transfer learning network # Use token-level output. - query = outputs["pooled_output"] - left = outputs['pooled_output_2'] + query = outputs["sequence_output"] + left = outputs['sequence_output_2'] # Select fine-tune strategy strategy = hub.AdamWeightDecayStrategy( diff --git a/demo/pointwise_text_matching/ernie_pointwise_matching_predict.py b/demo/pointwise_text_matching/ernie_pointwise_matching_predict.py index 18d75a98..26109cb6 100644 --- a/demo/pointwise_text_matching/ernie_pointwise_matching_predict.py +++ b/demo/pointwise_text_matching/ernie_pointwise_matching_predict.py @@ -48,8 +48,8 @@ if __name__ == '__main__': # Construct transfer learning network # Use token-level output. - query = outputs["pooled_output"] - left = outputs['pooled_output_2'] + query = outputs["sequence_output"] + left = outputs['sequence_output_2'] # Select fine-tune strategy strategy = hub.AdamWeightDecayStrategy() @@ -73,12 +73,12 @@ if __name__ == '__main__': # Prediction data sample. text_pairs = [ [ - "小品《战狼故事》中，吴京突破重重障碍解救爱人，深情告白太感人;爱人", # query - "外文名:愛人;摘要:爱人，意思是：情人。;义项描述:日本语词汇;语言:日文;中文名:爱人;标签:文化;", # title + "淘宝上怎么用信用卡分期付款", # query + "淘宝上怎么分期付款，没有信用卡", # title ], [ - "儿子祝融被杀害，西天王大发雷霆，立即下令捉拿天庭三公主;儿子", # query - "摘要:《儿子》是曹国昌1983年创作的木雕，收藏于中国美术馆。;材质：:木雕;作者：:曹国昌;中文名:儿子;创作年代：:1983年;义项描述:曹国昌木雕;标签:文化;", # title + "山楂干怎么吃好吃？", # query + "山楂怎么做好吃", # title ] ] @@ -88,6 +88,6 @@ if __name__ == '__main__': max_seq_len=args.max_seq_len, label_list=dataset.get_labels(), return_result=True, - accelerate_mode=False) + accelerate_mode=True) for index, text in enumerate(text_pairs): print("data: %s, prediction_label: %s" % (text, results[index])) diff --git a/demo/pointwise_text_matching/run_embedding_pointwise_matching.sh b/demo/pointwise_text_matching/run_embedding_pointwise_matching.sh index bd0f3e24..68c8cb73 100644 --- a/demo/pointwise_text_matching/run_embedding_pointwise_matching.sh +++ b/demo/pointwise_text_matching/run_embedding_pointwise_matching.sh @@ -5,5 +5,5 @@ python -u embedding_pointwise_matching.py \ --checkpoint_dir=${CKPT_DIR} \ --learning_rate=5e-3 \ --max_seq_len=128 \ - --num_epoch=300 \ + --num_epoch=100 \ --network=bow diff --git a/paddlehub/finetune/task/matching_task.py b/paddlehub/finetune/task/matching_task.py index 95862e44..5f4eb115 100644 --- a/paddlehub/finetune/task/matching_task.py +++ b/paddlehub/finetune/task/matching_task.py @@ -106,32 +106,81 @@ class PairwiseTextMatchingTask(BaseTask): % self.network) query_feats, left_feats = net_func(query_unpad, left_unpad) + query_feats = fluid.layers.fc( + input=query_feats, + size=300, + param_attr=fluid.ParamAttr( + name="query_fc_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02), + ), + bias_attr=fluid.ParamAttr( + name="query_fc_b", + initializer=fluid.initializer.Constant(0.), + ), + act="tanh") + left_feats = fluid.layers.fc( + input=left_feats, + size=300, + param_attr=fluid.ParamAttr( + name="title_fc_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02), + ), + bias_attr=fluid.ParamAttr( + name="title_fc_b", + initializer=fluid.initializer.Constant(0.), + ), + act="tanh") + left_concat = fluid.layers.concat( input=[query_feats, left_feats], axis=1) - query_feats, right_feats = net_func(query_unpad, right_unpad) + _, right_feats = net_func(query_unpad, right_unpad) + right_feats = fluid.layers.fc( + input=right_feats, + size=300, + param_attr=fluid.ParamAttr( + name="title_fc_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02), + ), + bias_attr=fluid.ParamAttr( + name="title_fc_b", + initializer=fluid.initializer.Constant(0.), + ), + act="tanh") + right_concat = fluid.layers.concat( - input=[query_feats, right_feats], - axis=1, - ) + input=[query_feats, right_feats], axis=1) + else: query_feats = fluid.layers.dropout( x=self.query_feature, dropout_prob=0.1, dropout_implementation="upscale_in_train") + query_mean = fluid.layers.reduce_mean(query_feats, dim=1) + left_feats = fluid.layers.dropout( x=self.left_feature, dropout_prob=0.1, dropout_implementation="upscale_in_train") + left_mean = fluid.layers.reduce_mean(left_feats, dim=1) + + left_sub = fluid.layers.elementwise_sub(query_mean, left_mean) + left_sub = fluid.layers.abs(left_sub) + left_concat = fluid.layers.concat( - input=[query_feats, left_feats], axis=-1) + input=[query_mean, left_mean, left_sub], axis=-1) right_feats = fluid.layers.dropout( x=self.right_feature, dropout_prob=0.1, dropout_implementation="upscale_in_train") + right_mean = fluid.layers.reduce_mean(right_feats, dim=1) + + right_sub = fluid.layers.elementwise_sub(query_mean, right_mean) + right_sub = fluid.layers.abs(right_sub) + right_concat = fluid.layers.concat( - input=[query_feats, right_feats], + input=[query_mean, right_mean, right_sub], axis=-1, ) @@ -435,22 +484,91 @@ class PointwiseTextMatchingTask(BaseTask): self.network) query_feats, title_feats = net_func(query_unpad, title_unpad) + + query_fc = fluid.layers.fc( + input=query_feats, + size=300, + param_attr=fluid.ParamAttr( + name="query_fc_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02), + ), + bias_attr=fluid.ParamAttr( + name="query_fc_b", + initializer=fluid.initializer.Constant(0.), + ), + act="tanh") + title_fc = fluid.layers.fc( + input=title_feats, + size=300, + param_attr=fluid.ParamAttr( + name="title_fc_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02), + ), + bias_attr=fluid.ParamAttr( + name="title_fc_b", + initializer=fluid.initializer.Constant(0.), + ), + act="tanh") title_concat = fluid.layers.concat( - input=[query_feats, title_feats], axis=1) + input=[query_fc, title_fc], axis=1) + + fc1 = fluid.layers.fc( + input=title_concat, + size=256, + param_attr=fluid.ParamAttr( + name="matching_fc1_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02), + ), + bias_attr=fluid.ParamAttr( + name="matching_fc1_b", + initializer=fluid.initializer.Constant(0.), + ), + act="tanh") + fc2 = fluid.layers.fc( + input=fc1, + size=128, + param_attr=fluid.ParamAttr( + name="matching_fc2_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02), + ), + bias_attr=fluid.ParamAttr( + name="matching_fc2_b", + initializer=fluid.initializer.Constant(0.), + ), + act="tanh") + projection = fluid.layers.fc( + input=fc2, + size=96, + param_attr=fluid.ParamAttr( + name="matching_fc3_w", + initializer=fluid.initializer.TruncatedNormal(scale=0.02), + ), + bias_attr=fluid.ParamAttr( + name="matching_fc3_b", + initializer=fluid.initializer.Constant(0.), + ), + act="tanh") + else: query_feats = fluid.layers.dropout( x=self.query_feature, dropout_prob=0.1, dropout_implementation="upscale_in_train") + query_mean = fluid.layers.reduce_mean(query_feats, dim=1) title_feats = fluid.layers.dropout( x=self.title_feature, dropout_prob=0.1, dropout_implementation="upscale_in_train") - title_concat = fluid.layers.concat( - input=[query_feats, title_feats], axis=-1) + title_mean = fluid.layers.reduce_mean(title_feats, dim=1) + + sub = fluid.layers.elementwise_sub(query_mean, title_mean) + sub = fluid.layers.abs(sub) + + projection = fluid.layers.concat( + input=[query_mean, title_mean, sub], axis=-1) score = fluid.layers.fc( - input=title_concat, + input=projection, size=2, param_attr=fluid.ParamAttr( name="matching_out_w", diff --git a/paddlehub/version.py b/paddlehub/version.py index 2a21a2de..37817839 100644 --- a/paddlehub/version.py +++ b/paddlehub/version.py @@ -13,5 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. """ PaddleHub version string """ -hub_version = "1.8.0" +hub_version = "1.8.1" module_proto_version = "1.0.0" -- GitLab