未验证 提交 e4a319d3 编写于 作者: S Steffy-zxf 提交者: GitHub

add ernie-skep (chnsenticorp) (#627)

上级 64a10def
......@@ -175,7 +175,7 @@ class EfficientNetB0ImageNet(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class FixResnext10132x48dwslImagenet(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
if not self.predictor_set:
......
......@@ -161,7 +161,7 @@ class MobileNetV2Animals(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class MobileNetV2Dishes(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -184,7 +184,7 @@ class MobileNetV2ImageNetSSLD(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class MobileNetV3Large(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class MobileNetV3Small(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class ResNet18vdImageNet(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
if not self.predictor_set:
......
......@@ -161,7 +161,7 @@ class ResNet50vdAnimals(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class ResNet50vdDishes(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class ResNet50vdDishes(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class ResNet50vdWildAnimals(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_data = list()
......
......@@ -161,7 +161,7 @@ class SEResNet18vdImageNet(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
if not self.predictor_set:
......
......@@ -83,7 +83,7 @@ class PyramidBoxFaceDetection(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -81,7 +81,7 @@ class PyramidBoxLiteMobile(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -107,7 +107,7 @@ class PyramidBoxLiteMobileMask(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -81,7 +81,7 @@ class PyramidBoxLiteServer(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -106,7 +106,7 @@ class PyramidBoxLiteServerMask(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -107,7 +107,7 @@ class FaceDetector320(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -106,7 +106,7 @@ class FaceDetector640(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -133,7 +133,7 @@ class FaceLandmarkLocalization(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# get all data
......
......@@ -323,7 +323,7 @@ class FasterRCNNResNet50(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
if data and 'image' in data:
......
......@@ -333,7 +333,7 @@ class FasterRCNNResNet50RPN(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -240,7 +240,7 @@ class RetinaNetResNet50FPN(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
all_images = list()
......
......@@ -194,7 +194,7 @@ class SSDMobileNetv1(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -200,7 +200,7 @@ class SSDVGG16_512(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -186,7 +186,7 @@ class YOLOv3DarkNet53Coco2017(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -199,7 +199,7 @@ class YOLOv3DarkNet53Pedestrian(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -199,7 +199,7 @@ class YOLOv3DarkNet53Vehicles(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -189,7 +189,7 @@ class YOLOv3MobileNetV1Coco2017(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -191,7 +191,7 @@ class YOLOv3ResNet34Coco2017(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -193,7 +193,7 @@ class YOLOv3ResNet50Coco2017(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
paths = paths if paths else list()
......
......@@ -86,7 +86,7 @@ class ACE2P(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -82,7 +82,7 @@ class DeeplabV3pXception65HumanSeg(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
# compatibility with older versions
......
......@@ -104,7 +104,7 @@ class StyleProjection(hub.Module):
int(_places[0])
except:
raise RuntimeError(
"Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
im_output = []
......
## 概述
SKEP(Sentiment Knowledge Enhanced Pre-training for Sentiment Analysis)是百度研究团队在2020年提出的基于情感知识增强的情感预训练算法,此算法采用无监督方法自动挖掘情感知识,然后利用情感知识构建预训练目标,从而让机器学会理解情感语义,在14项中英情感分析典型任务上全面超越SOTA,相关工作已经被ACL 2020录用。SKEP为各类情感分析任务提供统一且强大的情感语义表示。ernie_skep_sentiment_analysis Module可用于句子级情感分析任务预测。其在预训练时使用ERNIE 1.0 large预训练参数作为其网络参数初始化继续预训练。同时,该Module支持完成句子级情感分析任务迁移学习Fine-tune。
<p align="center">
<img src="https://bj.bcebos.com/paddlehub/model/nlp/skep.png" hspace='10'/> <br />
</p>
更多详情参考ACL 2020论文[SKEP: Sentiment Knowledge Enhanced Pre-training for Sentiment Analysis](https://arxiv.org/abs/2005.05635)
## 命令行预测
```shell
$ hub run ernie_skep_sentiment_analysis --input_text='虽然小明很努力,但是他还是没有考100分'
```
## API
```python
def predict_sentiment(texts=[], use_gpu=False)
```
预测API,分类输入文本的情感极性。
**参数**
* texts (list\[str\]): 待预测文本;
* use\_gpu (bool): 是否使用 GPU;**若使用GPU,请先设置CUDA_VISIBLE_DEVICES环境变量**
**返回**
* res (list\[dict\]): 情感分类结果的列表,列表中每一个元素为 dict,各字段为:
* text(str): 输入预测文本
* sentiment_label(str): 情感分类结果,或为positive或为negative
* positive_probs: 输入预测文本情感极性属于positive的概率
* negative_probs: 输入预测文本情感极性属于negative的概率
```python
def context(trainable=True, max_seq_len=128)
```
用于获取Module的上下文信息,得到输入、输出以及预训练的Paddle Program副本
**参数**
* trainable(bool): 设置为True时,Module中的参数在Fine-tune时也会随之训练,否则保持不变。
* max_seq_len(int): SKEP模型的最大序列长度,若序列长度不足,会通过padding方式补到**max_seq_len**, 若序列长度大于该值,则会以截断方式让序列长度为**max_seq_len**,max_seq_len可取值范围为0~512;
**返回**
* inputs: dict类型,各字段为:
* input_ids(Variable): Token Embedding,shape为\[batch_size, max_seq_len\],dtype为int64类型;
* position_id(Variable): Position Embedding,shape为\[batch_size, max_seq_len\],dtype为int64类型;
* segment_ids(Variable): Sentence Embedding,shape为\[batch_size, max_seq_len\],dtype为int64类型;
* input_mask(Variable): token是否为padding的标识,shape为\[batch_size, max_seq_len\],dtype为int64类型;
* outputs:dict类型,Module的输出特征,各字段为:
* pooled_output(Variable): 句子粒度的特征,可用于文本分类等任务,shape为 \[batch_size, 768\],dtype为int64类型;
* sequence_output(Variable): 字粒度的特征,可用于序列标注等任务,shape为 \[batch_size, seq_len, 768\],dtype为int64类型;
* program:包含该Module计算图的Program。
```python
def get_embedding(texts, use_gpu=False, batch_size=1)
```
用于获取输入文本的句子粒度特征与字粒度特征
**参数**
* texts(list):输入文本列表,格式为\[\[sample\_a\_text\_a, sample\_a\_text\_b\], \[sample\_b\_text\_a, sample\_b\_text\_b\],…,\],其中每个元素都是一个样例,每个样例可以包含text\_a与text\_b。
* use_gpu(bool):是否使用gpu,默认为False。对于GPU用户,建议开启use_gpu。**若使用GPU,请先设置CUDA_VISIBLE_DEVICES环境变量**
**返回**
* results(list): embedding特征,格式为\[\[sample\_a\_pooled\_feature, sample\_a\_seq\_feature\], \[sample\_b\_pooled\_feature, sample\_b\_seq\_feature\],…,\],其中每个元素都是对应样例的特征输出,每个样例都有句子粒度特征pooled\_feature与字粒度特征seq\_feature。
```python
def get_params_layer()
```
用于获取参数层信息,该方法与ULMFiTStrategy联用可以严格按照层数设置分层学习率与逐层解冻。
**参数**
*
**返回**
* params_layer(dict): key为参数名,值为参数所在层数
**代码示例**
情感极性预测代码示例:
```python
import paddlehub as hub
# Load ernie_skep_sentiment_analysis module.
module = hub.Module(name="ernie_skep_sentiment_analysis")
# Predict sentiment label
test_texts = ['你不是不聪明,而是不认真', '虽然小明很努力,但是他还是没有考100分']
results = module.predict_sentiment(test_texts, use_gpu=False)
```
## 服务部署
PaddleHub Serving 可以部署一个目标检测的在线服务。
### 第一步:启动PaddleHub Serving
运行启动命令:
```shell
$ hub serving start -m ernie_skep_sentiment_analysis
```
这样就完成了一个目标检测的服务化API的部署,默认端口号为8866。
**NOTE:** 如使用GPU预测,则需要在启动服务之前,请设置CUDA\_VISIBLE\_DEVICES环境变量,否则不用设置。
### 第二步:发送预测请求
配置好服务端,以下数行代码即可实现发送预测请求,获取预测结果
```python
import requests
import json
# 发送HTTP请求
data = {'texts':['你不是不聪明,而是不认真', '虽然小明很努力,但是他还是没有考100分']}
headers = {"Content-type": "application/json"}
url = "http://127.0.0.1:8866/predict/ernie_skep_sentiment_analysis"
r = requests.post(url=url, headers=headers, data=json.dumps(data))
# 打印预测结果
print(r.json()["results"])
```
## 查看代码
https://github.com/baidu/Senta
### 依赖
paddlepaddle >= 1.8.0
paddlehub >= 1.7.0
## 更新历史
* 1.0.0
初始发布
{
"attention_probs_dropout_prob": 0.1,
"hidden_act": "relu",
"hidden_dropout_prob": 0.1,
"hidden_size": 1024,
"initializer_range": 0.02,
"max_position_embeddings": 512,
"num_attention_heads": 16,
"num_hidden_layers": 24,
"sent_type_vocab_size": 4,
"task_type_vocab_size": 16,
"vocab_size": 12800,
"use_task_id": false
}
# -*- coding:utf-8 -**
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""ERNIE"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import json
import logging
import paddle.fluid as fluid
import six
from .transformer_encoder import encoder, pre_process_layer
from .transformer_encoder import gelu
class ErnieModel(object):
"""
ErnieModel
"""
def __init__(self,
src_ids,
position_ids,
sentence_ids,
input_mask,
config,
weight_sharing=True,
use_fp16=False):
"""
:param src_ids:
:param position_ids:
:param sentence_ids:
:param input_mask:
:param config:
:param weight_sharing:
:param use_fp16:
"""
self._hidden_size = config.get('hidden_size', 768)
self._emb_size = config.get('emb_size', self._hidden_size)
self._n_layer = config.get('num_hidden_layers', 12)
self._n_head = config.get('num_attention_heads', 12)
self._voc_size = config.get('vocab_size', 30522)
self._max_position_seq_len = config.get('max_position_embeddings', 512)
self._param_share = config.get('param_share', "normal")
self._pre_encoder_cmd = config.get('pre_encoder_cmd', "nd")
self._preprocess_cmd = config.get('preprocess_cmd', "")
self._postprocess_cmd = config.get('postprocess_cmd', "dan")
self._epsilon = config.get('epsilon', 1e-05)
self._emb_mapping_in = config.get('emb_mapping_in', False)
self._n_layer_per_block = config.get('n_layer_per_block', 1)
if config.has('sent_type_vocab_size'):
self._sent_types = config['sent_type_vocab_size']
else:
self._sent_types = config.get('type_vocab_size', 2)
self._use_sentence_id = config.get('use_sentence_id', True)
self._use_task_id = config.get('use_task_id', False)
if self._use_task_id:
self._task_types = config.get('task_type_vocab_size', 3)
self._hidden_act = config.get('hidden_act', 'gelu')
self._prepostprocess_dropout = config.get('hidden_dropout_prob', 0.1)
self._attention_dropout = config.get('attention_probs_dropout_prob',
0.1)
self._weight_sharing = weight_sharing
self._word_emb_name = "word_embedding"
self._pos_emb_name = "pos_embedding"
self._sent_emb_name = "sent_embedding"
self._task_emb_name = "task_embedding"
self._dtype = "float16" if use_fp16 else "float32"
self._emb_dtype = "float32"
# Initialize all weigths by truncated normal initializer, and all biases
# will be initialized by constant zero by default.
self._param_initializer = fluid.initializer.TruncatedNormal(
scale=config.get('initializer_range', 0.02))
self._build_model(src_ids, position_ids, sentence_ids, input_mask)
def _build_model(self, src_ids, position_ids, sentence_ids, input_mask):
"""
:param src_ids:
:param position_ids:
:param sentence_ids:
:param input_mask:
:return:
"""
# padding id in vocabulary must be set to 0
emb_out = fluid.layers.embedding(
input=src_ids,
dtype=self._emb_dtype,
size=[self._voc_size, self._emb_size],
param_attr=fluid.ParamAttr(
name=self._word_emb_name, initializer=self._param_initializer),
is_sparse=False)
position_emb_out = fluid.layers.embedding(
input=position_ids,
dtype=self._emb_dtype,
size=[self._max_position_seq_len, self._emb_size],
param_attr=fluid.ParamAttr(
name=self._pos_emb_name, initializer=self._param_initializer))
emb_out = emb_out + position_emb_out
if self._use_sentence_id:
sent_emb_out = fluid.layers.embedding(
sentence_ids,
dtype=self._emb_dtype,
size=[self._sent_types, self._emb_size],
param_attr=fluid.ParamAttr(
name=self._sent_emb_name,
initializer=self._param_initializer))
emb_out = emb_out + sent_emb_out
emb_out = pre_process_layer(
emb_out,
self._pre_encoder_cmd,
self._prepostprocess_dropout,
name='pre_encoder',
epsilon=self._epsilon)
if self._emb_mapping_in:
emb_out = fluid.layers.fc(
input=emb_out,
num_flatten_dims=2,
size=self._hidden_size,
param_attr=fluid.ParamAttr(
name='emb_hidden_mapping',
initializer=self._param_initializer),
bias_attr='emb_hidden_mapping_bias')
if self._dtype == "float16":
emb_out = fluid.layers.cast(x=emb_out, dtype=self._dtype)
input_mask = fluid.layers.cast(x=input_mask, dtype=self._dtype)
self_attn_mask = fluid.layers.matmul(
x=input_mask, y=input_mask, transpose_y=True)
self_attn_mask = fluid.layers.scale(
x=self_attn_mask, scale=10000.0, bias=-1.0, bias_after_scale=False)
n_head_self_attn_mask = fluid.layers.stack(
x=[self_attn_mask] * self._n_head, axis=1)
n_head_self_attn_mask.stop_gradient = True
self._enc_out, self._checkpoints = encoder(
enc_input=emb_out,
attn_bias=n_head_self_attn_mask,
n_layer=self._n_layer,
n_head=self._n_head,
d_key=self._hidden_size // self._n_head,
d_value=self._hidden_size // self._n_head,
d_model=self._hidden_size,
d_inner_hid=self._hidden_size * 4,
prepostprocess_dropout=self._prepostprocess_dropout,
attention_dropout=self._attention_dropout,
relu_dropout=0,
hidden_act=self._hidden_act,
preprocess_cmd=self._preprocess_cmd,
postprocess_cmd=self._postprocess_cmd,
param_initializer=self._param_initializer,
name='encoder',
param_share=self._param_share,
epsilon=self._epsilon,
n_layer_per_block=self._n_layer_per_block)
if self._dtype == "float16":
self._enc_out = fluid.layers.cast(
x=self._enc_out, dtype=self._emb_dtype)
def get_sequence_output(self):
"""
:return:
"""
return self._enc_out
def get_pooled_output(self):
"""Get the first feature of each sequence for classification"""
next_sent_feat = fluid.layers.slice(
input=self._enc_out, axes=[1], starts=[0], ends=[1])
"""
if self._dtype == "float16":
next_sent_feat = fluid.layers.cast(
x=next_sent_feat, dtype=self._emb_dtype)
next_sent_feat = fluid.layers.fc(
input=next_sent_feat,
size=self._emb_size,
param_attr=fluid.ParamAttr(
name="mask_lm_trans_fc.w_0", initializer=self._param_initializer),
bias_attr="mask_lm_trans_fc.b_0")
"""
"""
next_sent_feat = fluid.layers.fc(
input=next_sent_feat,
size=self._emb_size,
param_attr=fluid.ParamAttr(
name="mask_lm_trans_fc.w_0", initializer=self._param_initializer),
bias_attr="mask_lm_trans_fc.b_0")
"""
next_sent_feat = fluid.layers.fc(
input=next_sent_feat,
size=self._hidden_size,
act="tanh",
param_attr=fluid.ParamAttr(
name="pooled_fc.w_0", initializer=self._param_initializer),
bias_attr="pooled_fc.b_0")
return next_sent_feat
def get_lm_output(self, mask_label, mask_pos):
"""Get the loss & accuracy for pretraining"""
mask_pos = fluid.layers.cast(x=mask_pos, dtype='int32')
# extract the first token feature in each sentence
self.next_sent_feat = self.get_pooled_output()
reshaped_emb_out = fluid.layers.reshape(
x=self._enc_out, shape=[-1, self._hidden_size])
# extract masked tokens' feature
mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos)
if self._dtype == "float16":
mask_feat = fluid.layers.cast(x=mask_feat, dtype=self._emb_dtype)
# transform: fc
if self._hidden_act == 'gelu' or self._hidden_act == 'gelu.precise':
_hidden_act = 'gelu'
elif self._hidden_act == 'gelu.approximate':
_hidden_act = None
else:
_hidden_act = self._hidden_act
mask_trans_feat = fluid.layers.fc(
input=mask_feat,
size=self._emb_size,
act=_hidden_act,
param_attr=fluid.ParamAttr(
name='mask_lm_trans_fc.w_0',
initializer=self._param_initializer),
bias_attr=fluid.ParamAttr(name='mask_lm_trans_fc.b_0'))
if self._hidden_act == 'gelu.approximate':
mask_trans_feat = gelu(mask_trans_feat)
else:
pass
# transform: layer norm
mask_trans_feat = fluid.layers.layer_norm(
mask_trans_feat,
begin_norm_axis=len(mask_trans_feat.shape) - 1,
param_attr=fluid.ParamAttr(
name='mask_lm_trans_layer_norm_scale',
initializer=fluid.initializer.Constant(1.)),
bias_attr=fluid.ParamAttr(
name='mask_lm_trans_layer_norm_bias',
initializer=fluid.initializer.Constant(1.)))
# transform: layer norm
# mask_trans_feat = pre_process_layer(
# mask_trans_feat, 'n', name='mask_lm_trans')
mask_lm_out_bias_attr = fluid.ParamAttr(
name="mask_lm_out_fc.b_0",
initializer=fluid.initializer.Constant(value=0.0))
if self._weight_sharing:
fc_out = fluid.layers.matmul(
x=mask_trans_feat,
y=fluid.default_main_program().global_block().var(
self._word_emb_name),
transpose_y=True)
fc_out += fluid.layers.create_parameter(
shape=[self._voc_size],
dtype=self._emb_dtype,
attr=mask_lm_out_bias_attr,
is_bias=True)
else:
fc_out = fluid.layers.fc(
input=mask_trans_feat,
size=self._voc_size,
param_attr=fluid.ParamAttr(
name="mask_lm_out_fc.w_0",
initializer=self._param_initializer),
bias_attr=mask_lm_out_bias_attr)
mask_lm_loss = fluid.layers.softmax_with_cross_entropy(
logits=fc_out, label=mask_label)
mean_mask_lm_loss = fluid.layers.mean(mask_lm_loss)
return mean_mask_lm_loss
def get_task_output(self, task, task_labels):
"""
:param task:
:param task_labels:
:return:
"""
task_fc_out = fluid.layers.fc(
input=self.next_sent_feat,
size=task["num_labels"],
param_attr=fluid.ParamAttr(
name=task["task_name"] + "_fc.w_0",
initializer=self._param_initializer),
bias_attr=task["task_name"] + "_fc.b_0")
task_loss, task_softmax = fluid.layers.softmax_with_cross_entropy(
logits=task_fc_out, label=task_labels, return_softmax=True)
task_acc = fluid.layers.accuracy(input=task_softmax, label=task_labels)
mean_task_loss = fluid.layers.mean(task_loss)
return mean_task_loss, task_acc
class ErnieConfig(object):
"""parse ernie config"""
def __init__(self, config_path):
"""
:param config_path:
"""
self._config_dict = self._parse(config_path)
def _parse(self, config_path):
"""
:param config_path:
:return:
"""
try:
with open(config_path, 'r') as json_file:
config_dict = json.load(json_file)
except Exception:
raise IOError(
"Error in parsing Ernie model config file '%s'" % config_path)
else:
return config_dict
def __getitem__(self, key):
"""
:param key:
:return:
"""
return self._config_dict.get(key, None)
def has(self, key):
"""
:param key:
:return:
"""
if key in self._config_dict:
return True
return False
def get(self, key, default_value):
"""
:param key,default_value:
:retrun:
"""
if key in self._config_dict:
return self._config_dict[key]
else:
return default_value
def print_config(self):
"""
:return:
"""
for arg, value in sorted(six.iteritems(self._config_dict)):
logging.info('%s: %s' % (arg, value))
logging.info('------------------------------------------------')
# -*- coding:utf-8 -**
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformer encoder."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from functools import partial
import paddle.fluid as fluid
import paddle.fluid.layers as layers
import numpy as np
def gelu(x):
"""Gaussian Error Linear Unit.
This is a smoother version of the RELU.
Original paper: https://arxiv.org/abs/1606.08415
Args:
x: float Tensor to perform activation.
Returns:
`x` with the GELU activation applied.
"""
cdf = 0.5 * (1.0 + fluid.layers.tanh(
(np.sqrt(2.0 / np.pi) * (x + 0.044715 * fluid.layers.pow(x, 3.0)))))
return x * cdf
def multi_head_attention(queries,
keys,
values,
attn_bias,
d_key,
d_value,
d_model,
n_head=1,
dropout_rate=0.,
cache=None,
param_initializer=None,
name='multi_head_att'):
"""
Multi-Head Attention. Note that attn_bias is added to the logit before
computing softmax activiation to mask certain selected positions so that
they will not considered in attention weights.
"""
keys = queries if keys is None else keys
values = keys if values is None else values
if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3):
raise ValueError(
"Inputs: quries, keys and values should all be 3-D tensors. but {} v.s. {} v.s. {}"\
.format(queries.shape, keys.shape, values.shape))
def __compute_qkv(queries, keys, values, n_head, d_key, d_value):
"""
Add linear projection to queries, keys, and values.
"""
q = layers.fc(
input=queries,
size=d_key * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(
name=name + '_query_fc.w_0', initializer=param_initializer),
bias_attr=name + '_query_fc.b_0')
k = layers.fc(
input=keys,
size=d_key * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(
name=name + '_key_fc.w_0', initializer=param_initializer),
bias_attr=name + '_key_fc.b_0')
v = layers.fc(
input=values,
size=d_value * n_head,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(
name=name + '_value_fc.w_0', initializer=param_initializer),
bias_attr=name + '_value_fc.b_0')
return q, k, v
def __split_heads(x, n_head):
"""
Reshape the last dimension of inpunt tensor x so that it becomes two
dimensions and then transpose. Specifically, input a tensor with shape
[bs, max_sequence_length, n_head * hidden_dim] then output a tensor
with shape [bs, n_head, max_sequence_length, hidden_dim].
"""
hidden_size = x.shape[-1]
# The value 0 in shape attr means copying the corresponding dimension
# size of the input as the output dimension size.
reshaped = layers.reshape(
x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True)
# permuate the dimensions into:
# [batch_size, n_head, max_sequence_len, hidden_size_per_head]
return layers.transpose(x=reshaped, perm=[0, 2, 1, 3])
def __combine_heads(x):
"""
Transpose and then reshape the last two dimensions of inpunt tensor x
so that it becomes one dimension, which is reverse to __split_heads.
"""
if len(x.shape) == 3: return x
if len(x.shape) != 4:
raise ValueError("Input(x) should be a 4-D Tensor.")
trans_x = layers.transpose(x, perm=[0, 2, 1, 3])
# The value 0 in shape attr means copying the corresponding dimension
# size of the input as the output dimension size.
return layers.reshape(
x=trans_x,
shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]],
inplace=True)
def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate):
"""
Scaled Dot-Product Attention
"""
scaled_q = layers.scale(x=q, scale=d_key**-0.5)
product = layers.matmul(x=scaled_q, y=k, transpose_y=True)
if attn_bias:
product += attn_bias
weights = layers.softmax(product)
if dropout_rate:
weights = layers.dropout(
weights,
dropout_prob=dropout_rate,
dropout_implementation="upscale_in_train",
is_test=False)
out = layers.matmul(weights, v)
return out
q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value)
if cache is not None: # use cache and concat time steps
# Since the inplace reshape in __split_heads changes the shape of k and
# v, which is the cache input for next time step, reshape the cache
# input from the previous time step first.
k = cache["k"] = layers.concat(
[layers.reshape(cache["k"], shape=[0, 0, d_model]), k], axis=1)
v = cache["v"] = layers.concat(
[layers.reshape(cache["v"], shape=[0, 0, d_model]), v], axis=1)
q = __split_heads(q, n_head)
k = __split_heads(k, n_head)
v = __split_heads(v, n_head)
ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key,
dropout_rate)
out = __combine_heads(ctx_multiheads)
# Project back to the model size.
proj_out = layers.fc(
input=out,
size=d_model,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(
name=name + '_output_fc.w_0', initializer=param_initializer),
bias_attr=name + '_output_fc.b_0')
return proj_out
def positionwise_feed_forward(x,
d_inner_hid,
d_hid,
dropout_rate,
hidden_act,
param_initializer=None,
name='ffn'):
"""
Position-wise Feed-Forward Networks.
This module consists of two linear transformations with a ReLU activation
in between, which is applied to each position separately and identically.
"""
if hidden_act == 'gelu' or hidden_act == 'gelu.precise':
_hidden_act = 'gelu'
elif hidden_act == 'gelu.approximate':
_hidden_act = None
else:
_hidden_act = hidden_act
hidden = layers.fc(
input=x,
size=d_inner_hid,
num_flatten_dims=2,
act=_hidden_act,
param_attr=fluid.ParamAttr(
name=name + '_fc_0.w_0', initializer=param_initializer),
bias_attr=name + '_fc_0.b_0')
if hidden_act == 'gelu.approximate':
hidden = gelu(hidden)
if dropout_rate:
hidden = layers.dropout(
hidden,
dropout_prob=dropout_rate,
dropout_implementation="upscale_in_train",
is_test=False)
out = layers.fc(
input=hidden,
size=d_hid,
num_flatten_dims=2,
param_attr=fluid.ParamAttr(
name=name + '_fc_1.w_0', initializer=param_initializer),
bias_attr=name + '_fc_1.b_0')
return out
def pre_post_process_layer(prev_out,
out,
process_cmd,
dropout_rate=0.,
epsilon=1e-12,
name=''):
"""
Add residual connection, layer normalization and droput to the out tensor
optionally according to the value of process_cmd.
This will be used before or after multi-head attention and position-wise
feed-forward networks.
"""
for cmd in process_cmd:
if cmd == "a": # add residual connection
out = out + prev_out if prev_out else out
elif cmd == "n": # add layer normalization
out_dtype = out.dtype
if out_dtype == fluid.core.VarDesc.VarType.FP16:
out = layers.cast(x=out, dtype="float32")
out = layers.layer_norm(
out,
begin_norm_axis=len(out.shape) - 1,
param_attr=fluid.ParamAttr(
name=name + '_layer_norm_scale',
initializer=fluid.initializer.Constant(1.)),
bias_attr=fluid.ParamAttr(
name=name + '_layer_norm_bias',
initializer=fluid.initializer.Constant(0.)),
epsilon=epsilon)
if out_dtype == fluid.core.VarDesc.VarType.FP16:
out = layers.cast(x=out, dtype="float16")
elif cmd == "d": # add dropout
if dropout_rate:
out = layers.dropout(
out,
dropout_prob=dropout_rate,
dropout_implementation="upscale_in_train",
is_test=False)
return out
pre_process_layer = partial(pre_post_process_layer, None)
post_process_layer = pre_post_process_layer
def encoder_layer(
enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd="n",
postprocess_cmd="da",
param_initializer=None,
name='',
epsilon=1e-12,
):
"""The encoder layers that can be stacked to form a deep encoder.
This module consits of a multi-head (self) attention followed by
position-wise feed-forward networks and both the two components companied
with the post_process_layer to add residual connection, layer normalization
and droput.
"""
attn_output = multi_head_attention(
enc_input,
None,
None,
attn_bias,
d_key,
d_value,
d_model,
n_head,
attention_dropout,
param_initializer=param_initializer,
name=name + '_multi_head_att')
attn_output = post_process_layer(
enc_input,
attn_output,
postprocess_cmd,
prepostprocess_dropout,
name=name + '_post_att',
epsilon=epsilon)
ffd_output = positionwise_feed_forward(
attn_output,
d_inner_hid,
d_model,
relu_dropout,
hidden_act,
param_initializer=param_initializer,
name=name + '_ffn')
return post_process_layer(
attn_output,
ffd_output,
postprocess_cmd,
prepostprocess_dropout,
name=name + '_post_ffn',
epsilon=epsilon), ffd_output
def encoder_inner_share(enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd,
postprocess_cmd,
epsilon,
param_initializer=None,
name='',
n_layer_per_block=1):
"""
The encoder_inner_share is composed of n_layer_per_block layers returned by calling
encoder_layer.
"""
_checkpoints = []
for i in range(n_layer_per_block):
enc_output, cp = encoder_layer(
enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd,
postprocess_cmd,
param_initializer=param_initializer,
name=name + '_layer_' + str(i),
epsilon=epsilon,
)
_checkpoints.append(cp)
enc_input = enc_output
return enc_output, _checkpoints
def encoder_outer_share(enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd,
postprocess_cmd,
epsilon,
param_initializer=None,
name='',
n_layer_per_block=1):
"""
The encoder_outer_share is composed of n_layer_per_block layers returned by calling
encoder_layer.
"""
_checkpoints = []
for i in range(n_layer_per_block):
enc_output, cp = encoder_layer(
enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd,
postprocess_cmd,
param_initializer=param_initializer,
name=name,
epsilon=epsilon)
_checkpoints.append(cp)
enc_input = enc_output
return enc_output, _checkpoints
def encoder(enc_input,
attn_bias,
n_layer,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd,
postprocess_cmd,
epsilon,
n_layer_per_block,
param_initializer=None,
name='',
param_share=None):
"""
The encoder is composed of a stack of identical layers returned by calling
encoder_layer .
"""
checkpoints = []
# for outer_share it will share same param in one block,
# and for inner_share it will share param across blocks, rather than in one same block
#
# outer-share inner-share
# [1] [1] ----\ 1st block
# [1] [2] ----/
# [2] [1] ----\ 2nd block
# [2] [2] ----/
if param_share == "normal" or param_share == 'outer_share':
#n_layer_per_block=1, n_layer=24 for bert-large
#n_layer_per_block=1, n_layer=12 for bert-base
#n_layer_per_block=12, n_layer=12 for albert-xxlarge
#n_layer_per_block=6, n_layer=12 for albert-xxlarge-outershare
enc_fn = encoder_outer_share
name_fn = lambda i: name + '_layer_' + str(i)
elif param_share == "inner_share":
#n_layer_per_block = 2
enc_fn = encoder_inner_share
name_fn = lambda i: name
else:
raise ValueError('unsupported param share mode')
for i in range(n_layer // n_layer_per_block):
enc_output, cp = enc_fn(
enc_input,
attn_bias,
n_head,
d_key,
d_value,
d_model,
d_inner_hid,
prepostprocess_dropout,
attention_dropout,
relu_dropout,
hidden_act,
preprocess_cmd,
postprocess_cmd,
param_initializer=param_initializer,
name=name_fn(i),
n_layer_per_block=n_layer_per_block,
epsilon=epsilon,
)
checkpoints.extend(cp)
enc_input = enc_output
enc_output = pre_process_layer(
enc_output,
preprocess_cmd,
prepostprocess_dropout,
name="post_encoder",
epsilon=epsilon)
return enc_output, checkpoints
# coding:utf-8
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import ast
import os
from paddle.fluid.core import PaddleTensor, AnalysisConfig, create_paddle_predictor
from paddlehub import TransformerModule
from paddlehub.module.module import moduleinfo, runnable, serving
from paddlehub.reader.tokenization import convert_to_unicode, FullTokenizer
from paddlehub.reader.batching import pad_batch_data
import numpy as np
from ernie_skep_sentiment_analysis.model.ernie import ErnieModel, ErnieConfig
@moduleinfo(
name="ernie_skep_sentiment_analysis",
version="1.0.0",
summary=
"SKEP: Sentiment Knowledge Enhanced Pre-training for Sentiment Analysis. Ernie_skep_sentiment_analysis module is initialize with enie_1.0_chn_large when pretraining. This module is finetuned on ChnSentiCorp dataset to do sentiment claasification. It can do sentiment analysis prediction directly, label as positive or negative.",
author="baidu-nlp",
author_email="",
type="nlp/sentiment_analysis",
)
class ErnieSkepSentimentAnalysis(TransformerModule):
"""
Ernie_skep_sentiment_analysis module is initialize with enie_1.0_chn_large when pretraining.
This module is finetuned on ChnSentiCorp dataset to do sentiment claasification.
It can do sentiment analysis prediction directly, label as positive or negative.
"""
def _initialize(self):
ernie_config_path = os.path.join(self.directory, "assets",
"ernie_1.0_large_ch.config.json")
self.ernie_config = ErnieConfig(ernie_config_path)
self.MAX_SEQ_LEN = 512
self.vocab_path = os.path.join(self.directory, "assets",
"ernie_1.0_large_ch.vocab.txt")
self.params_path = os.path.join(self.directory, "assets", "params")
self.infer_model_path = os.path.join(self.directory, "assets",
"inference_step_601")
self.tokenizer = FullTokenizer(vocab_file=self.vocab_path)
self.vocab = self.tokenizer.vocab
self.pad_id = self.vocab["[PAD]"]
self.label_map = {0: 'negative', 1: 'positive'}
self._set_config()
def _set_config(self):
"""
predictor config setting
"""
model_file_path = os.path.join(self.infer_model_path, 'model')
params_file_path = os.path.join(self.infer_model_path, 'params')
config = AnalysisConfig(model_file_path, params_file_path)
try:
_places = os.environ["CUDA_VISIBLE_DEVICES"]
int(_places[0])
use_gpu = True
except:
use_gpu = False
if use_gpu:
config.enable_use_gpu(8000, 0)
else:
config.disable_gpu()
config.disable_glog_info()
self.predictor = create_paddle_predictor(config)
def net(self, input_ids, position_ids, segment_ids, input_mask):
"""
create neural network.
Args:
input_ids (tensor): the word ids.
position_ids (tensor): the position ids.
segment_ids (tensor): the segment ids.
input_mask (tensor): the padding mask.
Returns:
pooled_output (tensor): sentence-level output for classification task.
sequence_output (tensor): token-level output for sequence task.
"""
ernie = ErnieModel(
src_ids=input_ids,
position_ids=position_ids,
sentence_ids=segment_ids,
input_mask=input_mask,
config=self.ernie_config,
use_fp16=False)
pooled_output = ernie.get_pooled_output()
sequence_output = ernie.get_sequence_output()
return pooled_output, sequence_output
def array2tensor(self, arr_data):
"""
convert numpy array to PaddleTensor
"""
tensor_data = PaddleTensor(arr_data)
return tensor_data
@serving
def predict_sentiment(self, texts=[], use_gpu=False):
"""
Get the sentiment label for the predicted texts. It will be classified as positive and negative.
Args:
texts (list(str)): the data to be predicted.
use_gpu (bool): Whether to use gpu or not.
Returns:
res (list): The result of sentiment label and probabilties.
"""
if use_gpu:
try:
_places = os.environ["CUDA_VISIBLE_DEVICES"]
int(_places[0])
except:
raise RuntimeError(
"Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
)
results = []
for text in texts:
feature = self._convert_text_to_feature(text)
inputs = [self.array2tensor(ndarray) for ndarray in feature]
output = self.predictor.run(inputs)
probilities = np.array(output[0].data.float_data())
label = self.label_map[np.argmax(probilities)]
result = {
'text': text,
'sentiment_label': label,
'positive_probs': probilities[1],
'negative_probs': probilities[0]
}
results.append(result)
return results
def _convert_text_to_feature(self, text):
"""
Convert the raw text to feature which is needed to run program (feed_vars).
"""
text_a = convert_to_unicode(text)
tokens_a = self.tokenizer.tokenize(text_a)
max_seq_len = 512
# Account for [CLS] and [SEP] with "- 2"
if len(tokens_a) > max_seq_len - 2:
tokens_a = tokens_a[0:(max_seq_len - 2)]
tokens = []
text_type_ids = []
tokens.append("[CLS]")
text_type_ids.append(0)
for token in tokens_a:
tokens.append(token)
text_type_ids.append(0)
tokens.append("[SEP]")
text_type_ids.append(0)
token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
position_ids = list(range(len(token_ids)))
task_ids = [0] * len(token_ids)
padded_token_ids, input_mask = pad_batch_data([token_ids],
max_seq_len=max_seq_len,
pad_idx=self.pad_id,
return_input_mask=True)
padded_text_type_ids = pad_batch_data([text_type_ids],
max_seq_len=max_seq_len,
pad_idx=self.pad_id)
padded_position_ids = pad_batch_data([position_ids],
max_seq_len=max_seq_len,
pad_idx=self.pad_id)
padded_task_ids = pad_batch_data([task_ids],
max_seq_len=max_seq_len,
pad_idx=self.pad_id)
feature = [
padded_token_ids, padded_position_ids, padded_text_type_ids,
input_mask, padded_task_ids
]
return feature
@runnable
def run_cmd(self, argvs):
"""
Run as a command
"""
self.parser = argparse.ArgumentParser(
description="Run the %s module." % self.name,
prog='hub run %s' % self.name,
usage='%(prog)s',
add_help=True)
self.arg_input_group = self.parser.add_argument_group(
title="Input options", description="Input data. Required")
self.arg_config_group = self.parser.add_argument_group(
title="Config options",
description=
"Run configuration for controlling module behavior, not required.")
self.add_module_config_arg()
self.add_module_input_arg()
args = self.parser.parse_args(argvs)
results = self.predict_sentiment(
texts=[args.input_text], use_gpu=args.use_gpu)
return results
def add_module_config_arg(self):
"""
Add the command config options
"""
self.arg_config_group.add_argument(
'--use_gpu',
type=ast.literal_eval,
default=False,
help="whether use GPU or not")
def add_module_input_arg(self):
"""
Add the command input options
"""
self.arg_input_group.add_argument(
'--input_text', type=str, default=None, help="data to be predicted")
if __name__ == '__main__':
test_module = ErnieSkepSentimentAnalysis()
test_texts = ['你不是不聪明,而是不认真', '虽然小明很努力,但是他还是没有考100分']
results = test_module.predict_sentiment(test_texts, use_gpu=False)
print(results)
test_module.context(max_seq_len=128)
print(test_module.get_embedding(texts=[['你不是不聪明,而是不认真']]))
print(test_module.get_params_layer())
name: ernie_skep_sentiment_analysis
dir: "modules/text/sentiment_analysis/ernie_skep_sentiment_analysis"
exclude:
- README.md
resources:
-
url: https://paddlehub.bj.bcebos.com/model/nlp/ernie_skep_sentiment_analysis/assets.tar.gz
dest: assets
uncompress: True
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from unittest import TestCase, main
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
import numpy as np
import paddlehub as hub
class ErnieSkepSentimentAnalysisTestCase(TestCase):
def setUp(self):
self.module = hub.Module(name='ernie_skep_sentiment_analysis')
self.test_text = [[
'飞桨(PaddlePaddle)是国内开源产业级深度学习平台', 'PaddleHub是飞桨生态的预训练模型应用工具'
], ["飞浆PaddleHub"]]
self.test_data = ['你不是不聪明,而是不认真', '虽然小明很努力,但是他还是没有考100分']
self.results = [{
'text': '你不是不聪明,而是不认真',
'sentiment_label': 'negative',
'positive_probs': 0.10738213360309601,
'negative_probs': 0.8926178216934204
},
{
'text': '虽然小明很努力,但是他还是没有考100分',
'sentiment_label': 'negative',
'positive_probs': 0.053915347903966904,
'negative_probs': 0.9460846185684204
}]
def test_predict_sentiment(self):
results_1 = self.module.predict_sentiment(self.test_data, use_gpu=False)
results_2 = self.module.predict_sentiment(self.test_data, use_gpu=True)
for index, res in enumerate(results_1):
self.assertEqual(res['text'], self.results[index]['text'])
self.assertEqual(res['sentiment_label'],
self.results[index]['sentiment_label'])
self.assertTrue(
abs(res['positive_probs'] -
self.results[index]['positive_probs']) < 1e-6)
self.assertTrue(
abs(res['negative_probs'] -
self.results[index]['negative_probs']) < 1e-6)
self.assertEqual(res['text'], results_2[index]['text'])
self.assertEqual(res['sentiment_label'],
results_2[index]['sentiment_label'])
self.assertTrue(
abs(res['positive_probs'] -
results_2[index]['positive_probs']) < 1e-6)
self.assertTrue(
abs(res['negative_probs'] -
results_2[index]['negative_probs']) < 1e-6)
def test_get_embedding(self):
# test batch_size
max_seq_len = 128
results = self.module.get_embedding(
texts=self.test_text,
use_gpu=False,
batch_size=1,
max_seq_len=max_seq_len)
results_2 = self.module.get_embedding(
texts=self.test_text,
use_gpu=False,
batch_size=10,
max_seq_len=max_seq_len)
# 2 sample results
self.assertEqual(len(results), 2)
self.assertEqual(len(results_2), 2)
# sequence embedding and token embedding results per sample
self.assertEqual(len(results[0]), 2)
self.assertEqual(len(results_2[0]), 2)
# sequence embedding shape
self.assertEqual(results[0][0].shape, (1024, ))
self.assertEqual(results_2[0][0].shape, (1024, ))
# token embedding shape
self.assertEqual(results[0][1].shape, (max_seq_len, 1024))
self.assertEqual(results_2[0][1].shape, (max_seq_len, 1024))
# test gpu
results_3 = self.module.get_embedding(
texts=self.test_text,
use_gpu=True,
batch_size=1,
max_seq_len=max_seq_len)
diff = np.abs(results[0][0] - results_3[0][0])
self.assertTrue((diff < 1e-6).all)
diff = np.abs(results[0][1] - results_3[0][1])
self.assertTrue((diff < 1e-6).all)
diff = np.abs(results[1][0] - results_3[1][0])
self.assertTrue((diff < 1e-6).all)
diff = np.abs(results[1][1] - results_3[1][1])
self.assertTrue((diff < 1e-6).all)
def test_get_params_layer(self):
self.module.context()
layers = self.module.get_params_layer()
layers = list(set(layers.values()))
true_layers = [i for i in range(24)]
self.assertEqual(layers, true_layers)
def test_get_spm_path(self):
self.assertEqual(self.module.get_spm_path(), None)
def test_get_word_dict_path(self):
self.assertEqual(self.module.get_word_dict_path(), None)
def test_get_vocab_path(self):
vocab_path = self.module.get_vocab_path()
true_vocab_path = os.path.join(self.module.directory, "assets",
"ernie_1.0_large_ch.vocab.txt")
self.assertEqual(vocab_path, true_vocab_path)
if __name__ == '__main__':
main()
......@@ -319,8 +319,6 @@ class TransformerModule(NLPBaseModule):
pretraining_params_path,
main_program=main_program,
predicate=existed_params)
logger.info("Load pretraining parameters from {}.".format(
pretraining_params_path))
def param_prefix(self):
return "@HUB_%s@" % self.name
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册