Merge branch 'release/v1.7' of https://github.com/PaddlePaddle/PaddleHub into release/v1.7

hub_module/modules/image/classification/efficientnetb0_small_imagenet/module.py

@@ -175,7 +175,7 @@ class EfficientNetB0ImageNet(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/fix_resnext101_32x48d_wsl_imagenet/module.py

@@ -161,7 +161,7 @@ class FixResnext10132x48dwslImagenet(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         if not self.predictor_set:

hub_module/modules/image/classification/mobilenet_v2_animals/module.py

@@ -161,7 +161,7 @@ class MobileNetV2Animals(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/mobilenet_v2_dishes/module.py

@@ -161,7 +161,7 @@ class MobileNetV2Dishes(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/mobilenet_v2_imagenet_ssld/module.py

@@ -184,7 +184,7 @@ class MobileNetV2ImageNetSSLD(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/mobilenet_v3_large_imagenet_ssld/module.py

@@ -161,7 +161,7 @@ class MobileNetV3Large(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/mobilenet_v3_small_imagenet_ssld/module.py

@@ -161,7 +161,7 @@ class MobileNetV3Small(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/resnet18_vd_imagenet/module.py

@@ -161,7 +161,7 @@ class ResNet18vdImageNet(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         if not self.predictor_set:

hub_module/modules/image/classification/resnet50_vd_animals/module.py

@@ -161,7 +161,7 @@ class ResNet50vdAnimals(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/resnet50_vd_dishes/module.py

@@ -161,7 +161,7 @@ class ResNet50vdDishes(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/resnet50_vd_imagenet_ssld/module.py

@@ -161,7 +161,7 @@ class ResNet50vdDishes(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/resnet50_vd_wildanimals/module.py

@@ -161,7 +161,7 @@ class ResNet50vdWildAnimals(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_data = list()

hub_module/modules/image/classification/se_resnet18_vd_imagenet/module.py

@@ -161,7 +161,7 @@ class SEResNet18vdImageNet(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         if not self.predictor_set:

hub_module/modules/image/face_detection/pyramidbox_face_detection/module.py

@@ -83,7 +83,7 @@ class PyramidBoxFaceDetection(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/face_detection/pyramidbox_lite_mobile/module.py

@@ -81,7 +81,7 @@ class PyramidBoxLiteMobile(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/face_detection/pyramidbox_lite_mobile_mask/module.py

@@ -107,7 +107,7 @@ class PyramidBoxLiteMobileMask(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/face_detection/pyramidbox_lite_server/module.py

@@ -81,7 +81,7 @@ class PyramidBoxLiteServer(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/face_detection/pyramidbox_lite_server_mask/module.py

@@ -106,7 +106,7 @@ class PyramidBoxLiteServerMask(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/face_detection/ultra_light_fast_generic_face_detector_1mb_320/module.py

@@ -107,7 +107,7 @@ class FaceDetector320(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/face_detection/ultra_light_fast_generic_face_detector_1mb_640/module.py

@@ -106,7 +106,7 @@ class FaceDetector640(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/keypoint_detection/face_landmark_localization/module.py

@@ -133,7 +133,7 @@ class FaceLandmarkLocalization(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # get all data

hub_module/modules/image/object_detection/faster_rcnn_resnet50_coco2017/module.py

@@ -323,7 +323,7 @@ class FasterRCNNResNet50(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
         paths = paths if paths else list()
         if data and 'image' in data:

hub_module/modules/image/object_detection/faster_rcnn_resnet50_fpn_coco2017/module.py

@@ -333,7 +333,7 @@ class FasterRCNNResNet50RPN(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/object_detection/retinanet_resnet50_fpn_coco2017/module.py

@@ -240,7 +240,7 @@ class RetinaNetResNet50FPN(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         all_images = list()

hub_module/modules/image/object_detection/ssd_mobilenet_v1_pascal/module.py

@@ -194,7 +194,7 @@ class SSDMobileNetv1(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/object_detection/ssd_vgg16_512_coco2017/module.py

@@ -200,7 +200,7 @@ class SSDVGG16_512(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/object_detection/yolov3_darknet53_coco2017/module.py

@@ -186,7 +186,7 @@ class YOLOv3DarkNet53Coco2017(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/object_detection/yolov3_darknet53_pedestrian/module.py

@@ -199,7 +199,7 @@ class YOLOv3DarkNet53Pedestrian(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/object_detection/yolov3_darknet53_vehicles/module.py

@@ -199,7 +199,7 @@ class YOLOv3DarkNet53Vehicles(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/object_detection/yolov3_mobilenet_v1_coco2017/module.py

@@ -189,7 +189,7 @@ class YOLOv3MobileNetV1Coco2017(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/object_detection/yolov3_resnet34_coco2017/module.py

@@ -191,7 +191,7 @@ class YOLOv3ResNet34Coco2017(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/object_detection/yolov3_resnet50_vd_coco2017/module.py

@@ -193,7 +193,7 @@ class YOLOv3ResNet50Coco2017(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         paths = paths if paths else list()

hub_module/modules/image/semantic_segmentation/ace2p/module.py

@@ -86,7 +86,7 @@ class ACE2P(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/semantic_segmentation/deeplabv3p_xception65_humanseg/module.py

@@ -82,7 +82,7 @@ class DeeplabV3pXception65HumanSeg(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         # compatibility with older versions

hub_module/modules/image/style_transfer/stylepro_artistic/module.py

@@ -104,7 +104,7 @@ class StyleProjection(hub.Module):
                 int(_places[0])
             except:
                 raise RuntimeError(
-                    "Attempt to use GPU for prediction, but environment variable CUDA_VISIBLE_DEVICES was not set correctly."
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
                 )
 
         im_output = []

hub_module/modules/text/sentiment_analysis/ernie_skep_sentiment_analysis/README.md

+## 概述
+
+SKEP（Sentiment Knowledge Enhanced Pre-training for Sentiment Analysis）是百度研究团队在2020年提出的基于情感知识增强的情感预训练算法，此算法采用无监督方法自动挖掘情感知识，然后利用情感知识构建预训练目标，从而让机器学会理解情感语义，在14项中英情感分析典型任务上全面超越SOTA，相关工作已经被ACL 2020录用。SKEP为各类情感分析任务提供统一且强大的情感语义表示。ernie_skep_sentiment_analysis Module可用于句子级情感分析任务预测。其在预训练时使用ERNIE 1.0 large预训练参数作为其网络参数初始化继续预训练。同时，该Module支持完成句子级情感分析任务迁移学习Fine-tune。
+
+<p align="center">
+<img src="https://bj.bcebos.com/paddlehub/model/nlp/skep.png" hspace='10'/> <br />
+</p>
+
+更多详情参考ACL 2020论文[SKEP: Sentiment Knowledge Enhanced Pre-training for Sentiment Analysis](https://arxiv.org/abs/2005.05635)
+
+## 命令行预测
+
+```shell
+$ hub run ernie_skep_sentiment_analysis --input_text='虽然小明很努力，但是他还是没有考100分'
+```
+
+## API
+
+```python
+def predict_sentiment(texts=[], use_gpu=False)
+```
+
+预测API，分类输入文本的情感极性。
+
+**参数**
+
+* texts (list\[str\]): 待预测文本；
+* use\_gpu (bool): 是否使用 GPU；**若使用GPU，请先设置CUDA_VISIBLE_DEVICES环境变量**；
+
+**返回**
+
+* res (list\[dict\]): 情感分类结果的列表，列表中每一个元素为 dict，各字段为：
+    * text(str): 输入预测文本
+    * sentiment_label(str): 情感分类结果，或为positive或为negative
+    * positive_probs: 输入预测文本情感极性属于positive的概率
+    * negative_probs: 输入预测文本情感极性属于negative的概率
+
+```python
+def context(trainable=True, max_seq_len=128)
+```
+用于获取Module的上下文信息，得到输入、输出以及预训练的Paddle Program副本
+
+**参数**
+* trainable(bool): 设置为True时，Module中的参数在Fine-tune时也会随之训练，否则保持不变。
+* max_seq_len(int): SKEP模型的最大序列长度，若序列长度不足，会通过padding方式补到**max_seq_len**, 若序列长度大于该值，则会以截断方式让序列长度为**max_seq_len**，max_seq_len可取值范围为0～512；
+
+**返回**
+* inputs: dict类型，各字段为：
+  * input_ids(Variable): Token Embedding，shape为\[batch_size, max_seq_len\]，dtype为int64类型；
+  * position_id(Variable): Position Embedding，shape为\[batch_size, max_seq_len\]，dtype为int64类型；
+  * segment_ids(Variable): Sentence Embedding，shape为\[batch_size, max_seq_len\]，dtype为int64类型；
+  * input_mask(Variable): token是否为padding的标识，shape为\[batch_size, max_seq_len\]，dtype为int64类型；
+
+* outputs：dict类型，Module的输出特征，各字段为：
+  * pooled_output(Variable): 句子粒度的特征，可用于文本分类等任务，shape为 \[batch_size, 768\]，dtype为int64类型；
+  * sequence_output(Variable): 字粒度的特征，可用于序列标注等任务，shape为 \[batch_size, seq_len, 768\]，dtype为int64类型；
+
+* program：包含该Module计算图的Program。
+
+```python
+def get_embedding(texts, use_gpu=False, batch_size=1)
+```
+
+用于获取输入文本的句子粒度特征与字粒度特征
+
+**参数**
+
+* texts(list)：输入文本列表，格式为\[\[sample\_a\_text\_a, sample\_a\_text\_b\], \[sample\_b\_text\_a, sample\_b\_text\_b\],…,\]，其中每个元素都是一个样例，每个样例可以包含text\_a与text\_b。
+* use_gpu(bool)：是否使用gpu，默认为False。对于GPU用户，建议开启use_gpu。**若使用GPU，请先设置CUDA_VISIBLE_DEVICES环境变量**；
+
+**返回**
+
+* results(list): embedding特征，格式为\[\[sample\_a\_pooled\_feature, sample\_a\_seq\_feature\], \[sample\_b\_pooled\_feature, sample\_b\_seq\_feature\],…,\]，其中每个元素都是对应样例的特征输出，每个样例都有句子粒度特征pooled\_feature与字粒度特征seq\_feature。
+
+```python
+def get_params_layer()
+```
+
+用于获取参数层信息，该方法与ULMFiTStrategy联用可以严格按照层数设置分层学习率与逐层解冻。
+
+**参数**
+
+* 无
+
+**返回**
+
+* params_layer(dict): key为参数名，值为参数所在层数
+
+**代码示例**
+
+情感极性预测代码示例：
+
+```python
+import paddlehub as hub
+
+# Load ernie_skep_sentiment_analysis module.
+module = hub.Module(name="ernie_skep_sentiment_analysis")
+
+# Predict sentiment label
+test_texts = ['你不是不聪明，而是不认真', '虽然小明很努力，但是他还是没有考100分']
+results = module.predict_sentiment(test_texts, use_gpu=False)
+```
+
+## 服务部署
+
+PaddleHub Serving 可以部署一个目标检测的在线服务。
+
+### 第一步：启动PaddleHub Serving
+
+运行启动命令：
+```shell
+$ hub serving start -m ernie_skep_sentiment_analysis
+```
+
+这样就完成了一个目标检测的服务化API的部署，默认端口号为8866。
+
+**NOTE:** 如使用GPU预测，则需要在启动服务之前，请设置CUDA\_VISIBLE\_DEVICES环境变量，否则不用设置。
+
+### 第二步：发送预测请求
+
+配置好服务端，以下数行代码即可实现发送预测请求，获取预测结果
+
+```python
+import requests
+import json
+
+# 发送HTTP请求
+data = {'texts':['你不是不聪明，而是不认真', '虽然小明很努力，但是他还是没有考100分']}
+headers = {"Content-type": "application/json"}
+url = "http://127.0.0.1:8866/predict/ernie_skep_sentiment_analysis"
+r = requests.post(url=url, headers=headers, data=json.dumps(data))
+
+# 打印预测结果
+print(r.json()["results"])
+```
+
+## 查看代码
+
+https://github.com/baidu/Senta
+
+### 依赖
+
+paddlepaddle >= 1.8.0
+
+paddlehub >= 1.7.0
+
+
+## 更新历史
+
+* 1.0.0
+
+  初始发布

hub_module/modules/text/sentiment_analysis/ernie_skep_sentiment_analysis/assets/ernie_1.0_large_ch.config.json

+{
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "relu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "max_position_embeddings": 512,
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "sent_type_vocab_size": 4,
+  "task_type_vocab_size": 16,
+  "vocab_size": 12800,
+  "use_task_id": false
+}

hub_module/modules/text/sentiment_analysis/ernie_skep_sentiment_analysis/model/ernie.py

+# -*- coding:utf-8 -**
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ERNIE"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import json
+import logging
+
+import paddle.fluid as fluid
+import six
+
+from .transformer_encoder import encoder, pre_process_layer
+from .transformer_encoder import gelu
+
+
+class ErnieModel(object):
+    """
+    ErnieModel
+    """
+
+    def __init__(self,
+                 src_ids,
+                 position_ids,
+                 sentence_ids,
+                 input_mask,
+                 config,
+                 weight_sharing=True,
+                 use_fp16=False):
+        """
+        :param src_ids:
+        :param position_ids:
+        :param sentence_ids:
+        :param input_mask:
+        :param config:
+        :param weight_sharing:
+        :param use_fp16:
+        """
+        self._hidden_size = config.get('hidden_size', 768)
+        self._emb_size = config.get('emb_size', self._hidden_size)
+        self._n_layer = config.get('num_hidden_layers', 12)
+        self._n_head = config.get('num_attention_heads', 12)
+        self._voc_size = config.get('vocab_size', 30522)
+        self._max_position_seq_len = config.get('max_position_embeddings', 512)
+        self._param_share = config.get('param_share', "normal")
+        self._pre_encoder_cmd = config.get('pre_encoder_cmd', "nd")
+        self._preprocess_cmd = config.get('preprocess_cmd', "")
+        self._postprocess_cmd = config.get('postprocess_cmd', "dan")
+        self._epsilon = config.get('epsilon', 1e-05)
+        self._emb_mapping_in = config.get('emb_mapping_in', False)
+        self._n_layer_per_block = config.get('n_layer_per_block', 1)
+
+        if config.has('sent_type_vocab_size'):
+            self._sent_types = config['sent_type_vocab_size']
+        else:
+            self._sent_types = config.get('type_vocab_size', 2)
+
+        self._use_sentence_id = config.get('use_sentence_id', True)
+        self._use_task_id = config.get('use_task_id', False)
+        if self._use_task_id:
+            self._task_types = config.get('task_type_vocab_size', 3)
+        self._hidden_act = config.get('hidden_act', 'gelu')
+        self._prepostprocess_dropout = config.get('hidden_dropout_prob', 0.1)
+        self._attention_dropout = config.get('attention_probs_dropout_prob',
+                                             0.1)
+        self._weight_sharing = weight_sharing
+
+        self._word_emb_name = "word_embedding"
+        self._pos_emb_name = "pos_embedding"
+        self._sent_emb_name = "sent_embedding"
+        self._task_emb_name = "task_embedding"
+        self._dtype = "float16" if use_fp16 else "float32"
+        self._emb_dtype = "float32"
+        # Initialize all weigths by truncated normal initializer, and all biases
+        # will be initialized by constant zero by default.
+        self._param_initializer = fluid.initializer.TruncatedNormal(
+            scale=config.get('initializer_range', 0.02))
+
+        self._build_model(src_ids, position_ids, sentence_ids, input_mask)
+
+    def _build_model(self, src_ids, position_ids, sentence_ids, input_mask):
+        """
+        :param src_ids:
+        :param position_ids:
+        :param sentence_ids:
+        :param input_mask:
+        :return:
+        """
+        # padding id in vocabulary must be set to 0
+        emb_out = fluid.layers.embedding(
+            input=src_ids,
+            dtype=self._emb_dtype,
+            size=[self._voc_size, self._emb_size],
+            param_attr=fluid.ParamAttr(
+                name=self._word_emb_name, initializer=self._param_initializer),
+            is_sparse=False)
+
+        position_emb_out = fluid.layers.embedding(
+            input=position_ids,
+            dtype=self._emb_dtype,
+            size=[self._max_position_seq_len, self._emb_size],
+            param_attr=fluid.ParamAttr(
+                name=self._pos_emb_name, initializer=self._param_initializer))
+
+        emb_out = emb_out + position_emb_out
+
+        if self._use_sentence_id:
+            sent_emb_out = fluid.layers.embedding(
+                sentence_ids,
+                dtype=self._emb_dtype,
+                size=[self._sent_types, self._emb_size],
+                param_attr=fluid.ParamAttr(
+                    name=self._sent_emb_name,
+                    initializer=self._param_initializer))
+
+            emb_out = emb_out + sent_emb_out
+
+        emb_out = pre_process_layer(
+            emb_out,
+            self._pre_encoder_cmd,
+            self._prepostprocess_dropout,
+            name='pre_encoder',
+            epsilon=self._epsilon)
+
+        if self._emb_mapping_in:
+            emb_out = fluid.layers.fc(
+                input=emb_out,
+                num_flatten_dims=2,
+                size=self._hidden_size,
+                param_attr=fluid.ParamAttr(
+                    name='emb_hidden_mapping',
+                    initializer=self._param_initializer),
+                bias_attr='emb_hidden_mapping_bias')
+
+        if self._dtype == "float16":
+            emb_out = fluid.layers.cast(x=emb_out, dtype=self._dtype)
+            input_mask = fluid.layers.cast(x=input_mask, dtype=self._dtype)
+        self_attn_mask = fluid.layers.matmul(
+            x=input_mask, y=input_mask, transpose_y=True)
+
+        self_attn_mask = fluid.layers.scale(
+            x=self_attn_mask, scale=10000.0, bias=-1.0, bias_after_scale=False)
+        n_head_self_attn_mask = fluid.layers.stack(
+            x=[self_attn_mask] * self._n_head, axis=1)
+        n_head_self_attn_mask.stop_gradient = True
+
+        self._enc_out, self._checkpoints = encoder(
+            enc_input=emb_out,
+            attn_bias=n_head_self_attn_mask,
+            n_layer=self._n_layer,
+            n_head=self._n_head,
+            d_key=self._hidden_size // self._n_head,
+            d_value=self._hidden_size // self._n_head,
+            d_model=self._hidden_size,
+            d_inner_hid=self._hidden_size * 4,
+            prepostprocess_dropout=self._prepostprocess_dropout,
+            attention_dropout=self._attention_dropout,
+            relu_dropout=0,
+            hidden_act=self._hidden_act,
+            preprocess_cmd=self._preprocess_cmd,
+            postprocess_cmd=self._postprocess_cmd,
+            param_initializer=self._param_initializer,
+            name='encoder',
+            param_share=self._param_share,
+            epsilon=self._epsilon,
+            n_layer_per_block=self._n_layer_per_block)
+        if self._dtype == "float16":
+            self._enc_out = fluid.layers.cast(
+                x=self._enc_out, dtype=self._emb_dtype)
+
+    def get_sequence_output(self):
+        """
+        :return:
+        """
+        return self._enc_out
+
+    def get_pooled_output(self):
+        """Get the first feature of each sequence for classification"""
+        next_sent_feat = fluid.layers.slice(
+            input=self._enc_out, axes=[1], starts=[0], ends=[1])
+        """
+        if self._dtype == "float16":
+            next_sent_feat = fluid.layers.cast(
+                x=next_sent_feat, dtype=self._emb_dtype)
+
+        next_sent_feat = fluid.layers.fc(
+            input=next_sent_feat,
+            size=self._emb_size,
+            param_attr=fluid.ParamAttr(
+                name="mask_lm_trans_fc.w_0", initializer=self._param_initializer),
+            bias_attr="mask_lm_trans_fc.b_0")
+        """
+        """
+        next_sent_feat = fluid.layers.fc(
+            input=next_sent_feat,
+            size=self._emb_size,
+            param_attr=fluid.ParamAttr(
+                name="mask_lm_trans_fc.w_0", initializer=self._param_initializer),
+            bias_attr="mask_lm_trans_fc.b_0")
+
+        """
+        next_sent_feat = fluid.layers.fc(
+            input=next_sent_feat,
+            size=self._hidden_size,
+            act="tanh",
+            param_attr=fluid.ParamAttr(
+                name="pooled_fc.w_0", initializer=self._param_initializer),
+            bias_attr="pooled_fc.b_0")
+        return next_sent_feat
+
+    def get_lm_output(self, mask_label, mask_pos):
+        """Get the loss & accuracy for pretraining"""
+        mask_pos = fluid.layers.cast(x=mask_pos, dtype='int32')
+        # extract the first token feature in each sentence
+        self.next_sent_feat = self.get_pooled_output()
+        reshaped_emb_out = fluid.layers.reshape(
+            x=self._enc_out, shape=[-1, self._hidden_size])
+        # extract masked tokens' feature
+        mask_feat = fluid.layers.gather(input=reshaped_emb_out, index=mask_pos)
+
+        if self._dtype == "float16":
+            mask_feat = fluid.layers.cast(x=mask_feat, dtype=self._emb_dtype)
+
+        # transform: fc
+        if self._hidden_act == 'gelu' or self._hidden_act == 'gelu.precise':
+            _hidden_act = 'gelu'
+        elif self._hidden_act == 'gelu.approximate':
+            _hidden_act = None
+        else:
+            _hidden_act = self._hidden_act
+        mask_trans_feat = fluid.layers.fc(
+            input=mask_feat,
+            size=self._emb_size,
+            act=_hidden_act,
+            param_attr=fluid.ParamAttr(
+                name='mask_lm_trans_fc.w_0',
+                initializer=self._param_initializer),
+            bias_attr=fluid.ParamAttr(name='mask_lm_trans_fc.b_0'))
+        if self._hidden_act == 'gelu.approximate':
+            mask_trans_feat = gelu(mask_trans_feat)
+        else:
+            pass
+        # transform: layer norm
+        mask_trans_feat = fluid.layers.layer_norm(
+            mask_trans_feat,
+            begin_norm_axis=len(mask_trans_feat.shape) - 1,
+            param_attr=fluid.ParamAttr(
+                name='mask_lm_trans_layer_norm_scale',
+                initializer=fluid.initializer.Constant(1.)),
+            bias_attr=fluid.ParamAttr(
+                name='mask_lm_trans_layer_norm_bias',
+                initializer=fluid.initializer.Constant(1.)))
+        # transform: layer norm
+        # mask_trans_feat = pre_process_layer(
+        #    mask_trans_feat, 'n', name='mask_lm_trans')
+
+        mask_lm_out_bias_attr = fluid.ParamAttr(
+            name="mask_lm_out_fc.b_0",
+            initializer=fluid.initializer.Constant(value=0.0))
+        if self._weight_sharing:
+            fc_out = fluid.layers.matmul(
+                x=mask_trans_feat,
+                y=fluid.default_main_program().global_block().var(
+                    self._word_emb_name),
+                transpose_y=True)
+            fc_out += fluid.layers.create_parameter(
+                shape=[self._voc_size],
+                dtype=self._emb_dtype,
+                attr=mask_lm_out_bias_attr,
+                is_bias=True)
+
+        else:
+            fc_out = fluid.layers.fc(
+                input=mask_trans_feat,
+                size=self._voc_size,
+                param_attr=fluid.ParamAttr(
+                    name="mask_lm_out_fc.w_0",
+                    initializer=self._param_initializer),
+                bias_attr=mask_lm_out_bias_attr)
+
+        mask_lm_loss = fluid.layers.softmax_with_cross_entropy(
+            logits=fc_out, label=mask_label)
+        mean_mask_lm_loss = fluid.layers.mean(mask_lm_loss)
+
+        return mean_mask_lm_loss
+
+    def get_task_output(self, task, task_labels):
+        """
+        :param task:
+        :param task_labels:
+        :return:
+        """
+        task_fc_out = fluid.layers.fc(
+            input=self.next_sent_feat,
+            size=task["num_labels"],
+            param_attr=fluid.ParamAttr(
+                name=task["task_name"] + "_fc.w_0",
+                initializer=self._param_initializer),
+            bias_attr=task["task_name"] + "_fc.b_0")
+        task_loss, task_softmax = fluid.layers.softmax_with_cross_entropy(
+            logits=task_fc_out, label=task_labels, return_softmax=True)
+        task_acc = fluid.layers.accuracy(input=task_softmax, label=task_labels)
+        mean_task_loss = fluid.layers.mean(task_loss)
+        return mean_task_loss, task_acc
+
+
+class ErnieConfig(object):
+    """parse ernie config"""
+
+    def __init__(self, config_path):
+        """
+        :param config_path:
+        """
+        self._config_dict = self._parse(config_path)
+
+    def _parse(self, config_path):
+        """
+        :param config_path:
+        :return:
+        """
+        try:
+            with open(config_path, 'r') as json_file:
+                config_dict = json.load(json_file)
+        except Exception:
+            raise IOError(
+                "Error in parsing Ernie model config file '%s'" % config_path)
+        else:
+            return config_dict
+
+    def __getitem__(self, key):
+        """
+        :param key:
+        :return:
+        """
+        return self._config_dict.get(key, None)
+
+    def has(self, key):
+        """
+        :param key:
+        :return:
+        """
+        if key in self._config_dict:
+            return True
+        return False
+
+    def get(self, key, default_value):
+        """
+        :param key,default_value:
+        :retrun:
+        """
+        if key in self._config_dict:
+            return self._config_dict[key]
+        else:
+            return default_value
+
+    def print_config(self):
+        """
+        :return:
+        """
+        for arg, value in sorted(six.iteritems(self._config_dict)):
+            logging.info('%s: %s' % (arg, value))
+        logging.info('------------------------------------------------')

hub_module/modules/text/sentiment_analysis/ernie_skep_sentiment_analysis/model/transformer_encoder.py

+# -*- coding:utf-8 -**
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Transformer encoder."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from functools import partial
+
+import paddle.fluid as fluid
+import paddle.fluid.layers as layers
+import numpy as np
+
+
+def gelu(x):
+    """Gaussian Error Linear Unit.
+
+  This is a smoother version of the RELU.
+  Original paper: https://arxiv.org/abs/1606.08415
+  Args:
+    x: float Tensor to perform activation.
+
+  Returns:
+    `x` with the GELU activation applied.
+  """
+    cdf = 0.5 * (1.0 + fluid.layers.tanh(
+        (np.sqrt(2.0 / np.pi) * (x + 0.044715 * fluid.layers.pow(x, 3.0)))))
+    return x * cdf
+
+
+def multi_head_attention(queries,
+                         keys,
+                         values,
+                         attn_bias,
+                         d_key,
+                         d_value,
+                         d_model,
+                         n_head=1,
+                         dropout_rate=0.,
+                         cache=None,
+                         param_initializer=None,
+                         name='multi_head_att'):
+    """
+    Multi-Head Attention. Note that attn_bias is added to the logit before
+    computing softmax activiation to mask certain selected positions so that
+    they will not considered in attention weights.
+    """
+    keys = queries if keys is None else keys
+    values = keys if values is None else values
+    if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3):
+        raise ValueError(
+            "Inputs: quries, keys and values should all be 3-D tensors. but {} v.s. {} v.s. {}"\
+                    .format(queries.shape, keys.shape, values.shape))
+
+    def __compute_qkv(queries, keys, values, n_head, d_key, d_value):
+        """
+        Add linear projection to queries, keys, and values.
+        """
+        q = layers.fc(
+            input=queries,
+            size=d_key * n_head,
+            num_flatten_dims=2,
+            param_attr=fluid.ParamAttr(
+                name=name + '_query_fc.w_0', initializer=param_initializer),
+            bias_attr=name + '_query_fc.b_0')
+        k = layers.fc(
+            input=keys,
+            size=d_key * n_head,
+            num_flatten_dims=2,
+            param_attr=fluid.ParamAttr(
+                name=name + '_key_fc.w_0', initializer=param_initializer),
+            bias_attr=name + '_key_fc.b_0')
+        v = layers.fc(
+            input=values,
+            size=d_value * n_head,
+            num_flatten_dims=2,
+            param_attr=fluid.ParamAttr(
+                name=name + '_value_fc.w_0', initializer=param_initializer),
+            bias_attr=name + '_value_fc.b_0')
+        return q, k, v
+
+    def __split_heads(x, n_head):
+        """
+        Reshape the last dimension of inpunt tensor x so that it becomes two
+        dimensions and then transpose. Specifically, input a tensor with shape
+        [bs, max_sequence_length, n_head * hidden_dim] then output a tensor
+        with shape [bs, n_head, max_sequence_length, hidden_dim].
+        """
+        hidden_size = x.shape[-1]
+        # The value 0 in shape attr means copying the corresponding dimension
+        # size of the input as the output dimension size.
+        reshaped = layers.reshape(
+            x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True)
+
+        # permuate the dimensions into:
+        # [batch_size, n_head, max_sequence_len, hidden_size_per_head]
+        return layers.transpose(x=reshaped, perm=[0, 2, 1, 3])
+
+    def __combine_heads(x):
+        """
+        Transpose and then reshape the last two dimensions of inpunt tensor x
+        so that it becomes one dimension, which is reverse to __split_heads.
+        """
+        if len(x.shape) == 3: return x
+        if len(x.shape) != 4:
+            raise ValueError("Input(x) should be a 4-D Tensor.")
+
+        trans_x = layers.transpose(x, perm=[0, 2, 1, 3])
+        # The value 0 in shape attr means copying the corresponding dimension
+        # size of the input as the output dimension size.
+        return layers.reshape(
+            x=trans_x,
+            shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]],
+            inplace=True)
+
+    def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate):
+        """
+        Scaled Dot-Product Attention
+        """
+        scaled_q = layers.scale(x=q, scale=d_key**-0.5)
+        product = layers.matmul(x=scaled_q, y=k, transpose_y=True)
+        if attn_bias:
+            product += attn_bias
+        weights = layers.softmax(product)
+        if dropout_rate:
+            weights = layers.dropout(
+                weights,
+                dropout_prob=dropout_rate,
+                dropout_implementation="upscale_in_train",
+                is_test=False)
+        out = layers.matmul(weights, v)
+        return out
+
+    q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value)
+
+    if cache is not None:  # use cache and concat time steps
+        # Since the inplace reshape in __split_heads changes the shape of k and
+        # v, which is the cache input for next time step, reshape the cache
+        # input from the previous time step first.
+        k = cache["k"] = layers.concat(
+            [layers.reshape(cache["k"], shape=[0, 0, d_model]), k], axis=1)
+        v = cache["v"] = layers.concat(
+            [layers.reshape(cache["v"], shape=[0, 0, d_model]), v], axis=1)
+
+    q = __split_heads(q, n_head)
+    k = __split_heads(k, n_head)
+    v = __split_heads(v, n_head)
+
+    ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key,
+                                                  dropout_rate)
+
+    out = __combine_heads(ctx_multiheads)
+
+    # Project back to the model size.
+    proj_out = layers.fc(
+        input=out,
+        size=d_model,
+        num_flatten_dims=2,
+        param_attr=fluid.ParamAttr(
+            name=name + '_output_fc.w_0', initializer=param_initializer),
+        bias_attr=name + '_output_fc.b_0')
+    return proj_out
+
+
+def positionwise_feed_forward(x,
+                              d_inner_hid,
+                              d_hid,
+                              dropout_rate,
+                              hidden_act,
+                              param_initializer=None,
+                              name='ffn'):
+    """
+    Position-wise Feed-Forward Networks.
+    This module consists of two linear transformations with a ReLU activation
+    in between, which is applied to each position separately and identically.
+    """
+    if hidden_act == 'gelu' or hidden_act == 'gelu.precise':
+        _hidden_act = 'gelu'
+    elif hidden_act == 'gelu.approximate':
+        _hidden_act = None
+    else:
+        _hidden_act = hidden_act
+    hidden = layers.fc(
+        input=x,
+        size=d_inner_hid,
+        num_flatten_dims=2,
+        act=_hidden_act,
+        param_attr=fluid.ParamAttr(
+            name=name + '_fc_0.w_0', initializer=param_initializer),
+        bias_attr=name + '_fc_0.b_0')
+    if hidden_act == 'gelu.approximate':
+        hidden = gelu(hidden)
+
+    if dropout_rate:
+        hidden = layers.dropout(
+            hidden,
+            dropout_prob=dropout_rate,
+            dropout_implementation="upscale_in_train",
+            is_test=False)
+    out = layers.fc(
+        input=hidden,
+        size=d_hid,
+        num_flatten_dims=2,
+        param_attr=fluid.ParamAttr(
+            name=name + '_fc_1.w_0', initializer=param_initializer),
+        bias_attr=name + '_fc_1.b_0')
+    return out
+
+
+def pre_post_process_layer(prev_out,
+                           out,
+                           process_cmd,
+                           dropout_rate=0.,
+                           epsilon=1e-12,
+                           name=''):
+    """
+    Add residual connection, layer normalization and droput to the out tensor
+    optionally according to the value of process_cmd.
+    This will be used before or after multi-head attention and position-wise
+    feed-forward networks.
+    """
+    for cmd in process_cmd:
+        if cmd == "a":  # add residual connection
+            out = out + prev_out if prev_out else out
+        elif cmd == "n":  # add layer normalization
+            out_dtype = out.dtype
+            if out_dtype == fluid.core.VarDesc.VarType.FP16:
+                out = layers.cast(x=out, dtype="float32")
+            out = layers.layer_norm(
+                out,
+                begin_norm_axis=len(out.shape) - 1,
+                param_attr=fluid.ParamAttr(
+                    name=name + '_layer_norm_scale',
+                    initializer=fluid.initializer.Constant(1.)),
+                bias_attr=fluid.ParamAttr(
+                    name=name + '_layer_norm_bias',
+                    initializer=fluid.initializer.Constant(0.)),
+                epsilon=epsilon)
+            if out_dtype == fluid.core.VarDesc.VarType.FP16:
+                out = layers.cast(x=out, dtype="float16")
+        elif cmd == "d":  # add dropout
+            if dropout_rate:
+                out = layers.dropout(
+                    out,
+                    dropout_prob=dropout_rate,
+                    dropout_implementation="upscale_in_train",
+                    is_test=False)
+    return out
+
+
+pre_process_layer = partial(pre_post_process_layer, None)
+post_process_layer = pre_post_process_layer
+
+
+def encoder_layer(
+        enc_input,
+        attn_bias,
+        n_head,
+        d_key,
+        d_value,
+        d_model,
+        d_inner_hid,
+        prepostprocess_dropout,
+        attention_dropout,
+        relu_dropout,
+        hidden_act,
+        preprocess_cmd="n",
+        postprocess_cmd="da",
+        param_initializer=None,
+        name='',
+        epsilon=1e-12,
+):
+    """The encoder layers that can be stacked to form a deep encoder.
+    This module consits of a multi-head (self) attention followed by
+    position-wise feed-forward networks and both the two components companied
+    with the post_process_layer to add residual connection, layer normalization
+    and droput.
+    """
+
+    attn_output = multi_head_attention(
+        enc_input,
+        None,
+        None,
+        attn_bias,
+        d_key,
+        d_value,
+        d_model,
+        n_head,
+        attention_dropout,
+        param_initializer=param_initializer,
+        name=name + '_multi_head_att')
+
+    attn_output = post_process_layer(
+        enc_input,
+        attn_output,
+        postprocess_cmd,
+        prepostprocess_dropout,
+        name=name + '_post_att',
+        epsilon=epsilon)
+
+    ffd_output = positionwise_feed_forward(
+        attn_output,
+        d_inner_hid,
+        d_model,
+        relu_dropout,
+        hidden_act,
+        param_initializer=param_initializer,
+        name=name + '_ffn')
+
+    return post_process_layer(
+        attn_output,
+        ffd_output,
+        postprocess_cmd,
+        prepostprocess_dropout,
+        name=name + '_post_ffn',
+        epsilon=epsilon), ffd_output
+
+
+def encoder_inner_share(enc_input,
+                        attn_bias,
+                        n_head,
+                        d_key,
+                        d_value,
+                        d_model,
+                        d_inner_hid,
+                        prepostprocess_dropout,
+                        attention_dropout,
+                        relu_dropout,
+                        hidden_act,
+                        preprocess_cmd,
+                        postprocess_cmd,
+                        epsilon,
+                        param_initializer=None,
+                        name='',
+                        n_layer_per_block=1):
+    """
+       The encoder_inner_share is composed of n_layer_per_block layers returned by calling
+       encoder_layer.
+    """
+    _checkpoints = []
+    for i in range(n_layer_per_block):
+        enc_output, cp = encoder_layer(
+            enc_input,
+            attn_bias,
+            n_head,
+            d_key,
+            d_value,
+            d_model,
+            d_inner_hid,
+            prepostprocess_dropout,
+            attention_dropout,
+            relu_dropout,
+            hidden_act,
+            preprocess_cmd,
+            postprocess_cmd,
+            param_initializer=param_initializer,
+            name=name + '_layer_' + str(i),
+            epsilon=epsilon,
+        )
+        _checkpoints.append(cp)
+        enc_input = enc_output
+
+    return enc_output, _checkpoints
+
+
+def encoder_outer_share(enc_input,
+                        attn_bias,
+                        n_head,
+                        d_key,
+                        d_value,
+                        d_model,
+                        d_inner_hid,
+                        prepostprocess_dropout,
+                        attention_dropout,
+                        relu_dropout,
+                        hidden_act,
+                        preprocess_cmd,
+                        postprocess_cmd,
+                        epsilon,
+                        param_initializer=None,
+                        name='',
+                        n_layer_per_block=1):
+    """
+       The encoder_outer_share is composed of n_layer_per_block layers returned by calling
+       encoder_layer.
+    """
+    _checkpoints = []
+    for i in range(n_layer_per_block):
+        enc_output, cp = encoder_layer(
+            enc_input,
+            attn_bias,
+            n_head,
+            d_key,
+            d_value,
+            d_model,
+            d_inner_hid,
+            prepostprocess_dropout,
+            attention_dropout,
+            relu_dropout,
+            hidden_act,
+            preprocess_cmd,
+            postprocess_cmd,
+            param_initializer=param_initializer,
+            name=name,
+            epsilon=epsilon)
+        _checkpoints.append(cp)
+        enc_input = enc_output
+
+    return enc_output, _checkpoints
+
+
+def encoder(enc_input,
+            attn_bias,
+            n_layer,
+            n_head,
+            d_key,
+            d_value,
+            d_model,
+            d_inner_hid,
+            prepostprocess_dropout,
+            attention_dropout,
+            relu_dropout,
+            hidden_act,
+            preprocess_cmd,
+            postprocess_cmd,
+            epsilon,
+            n_layer_per_block,
+            param_initializer=None,
+            name='',
+            param_share=None):
+    """
+    The encoder is composed of a stack of identical layers returned by calling
+    encoder_layer .
+    """
+    checkpoints = []
+    # for outer_share it will share same param in one block,
+    # and for inner_share it will share param across blocks, rather than in one same block
+    #
+    # outer-share   inner-share
+    #    [1]           [1]      ----\ 1st block
+    #    [1]           [2]      ----/
+    #    [2]           [1]      ----\ 2nd block
+    #    [2]           [2]      ----/
+
+    if param_share == "normal" or param_share == 'outer_share':
+        #n_layer_per_block=1,  n_layer=24 for bert-large
+        #n_layer_per_block=1,  n_layer=12 for bert-base
+        #n_layer_per_block=12, n_layer=12 for albert-xxlarge
+        #n_layer_per_block=6,  n_layer=12 for albert-xxlarge-outershare
+        enc_fn = encoder_outer_share
+        name_fn = lambda i: name + '_layer_' + str(i)
+    elif param_share == "inner_share":
+        #n_layer_per_block = 2
+        enc_fn = encoder_inner_share
+        name_fn = lambda i: name
+    else:
+        raise ValueError('unsupported param share mode')
+
+    for i in range(n_layer // n_layer_per_block):
+        enc_output, cp = enc_fn(
+            enc_input,
+            attn_bias,
+            n_head,
+            d_key,
+            d_value,
+            d_model,
+            d_inner_hid,
+            prepostprocess_dropout,
+            attention_dropout,
+            relu_dropout,
+            hidden_act,
+            preprocess_cmd,
+            postprocess_cmd,
+            param_initializer=param_initializer,
+            name=name_fn(i),
+            n_layer_per_block=n_layer_per_block,
+            epsilon=epsilon,
+        )
+        checkpoints.extend(cp)
+        enc_input = enc_output
+    enc_output = pre_process_layer(
+        enc_output,
+        preprocess_cmd,
+        prepostprocess_dropout,
+        name="post_encoder",
+        epsilon=epsilon)
+
+    return enc_output, checkpoints

hub_module/modules/text/sentiment_analysis/ernie_skep_sentiment_analysis/module.py

+# coding:utf-8
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import ast
+import os
+
+from paddle.fluid.core import PaddleTensor, AnalysisConfig, create_paddle_predictor
+from paddlehub import TransformerModule
+from paddlehub.module.module import moduleinfo, runnable, serving
+from paddlehub.reader.tokenization import convert_to_unicode, FullTokenizer
+from paddlehub.reader.batching import pad_batch_data
+import numpy as np
+
+from ernie_skep_sentiment_analysis.model.ernie import ErnieModel, ErnieConfig
+
+
+@moduleinfo(
+    name="ernie_skep_sentiment_analysis",
+    version="1.0.0",
+    summary=
+    "SKEP: Sentiment Knowledge Enhanced Pre-training for Sentiment Analysis. Ernie_skep_sentiment_analysis module is initialize with enie_1.0_chn_large when pretraining. This module is finetuned on ChnSentiCorp dataset to do sentiment claasification. It can do sentiment analysis prediction directly, label as positive or negative.",
+    author="baidu-nlp",
+    author_email="",
+    type="nlp/sentiment_analysis",
+)
+class ErnieSkepSentimentAnalysis(TransformerModule):
+    """
+    Ernie_skep_sentiment_analysis module is initialize with enie_1.0_chn_large when pretraining.
+    This module is finetuned on ChnSentiCorp dataset to do sentiment claasification.
+    It can do sentiment analysis prediction directly, label as positive or negative.
+    """
+
+    def _initialize(self):
+        ernie_config_path = os.path.join(self.directory, "assets",
+                                         "ernie_1.0_large_ch.config.json")
+        self.ernie_config = ErnieConfig(ernie_config_path)
+        self.MAX_SEQ_LEN = 512
+        self.vocab_path = os.path.join(self.directory, "assets",
+                                       "ernie_1.0_large_ch.vocab.txt")
+        self.params_path = os.path.join(self.directory, "assets", "params")
+
+        self.infer_model_path = os.path.join(self.directory, "assets",
+                                             "inference_step_601")
+        self.tokenizer = FullTokenizer(vocab_file=self.vocab_path)
+
+        self.vocab = self.tokenizer.vocab
+        self.pad_id = self.vocab["[PAD]"]
+        self.label_map = {0: 'negative', 1: 'positive'}
+
+        self._set_config()
+
+    def _set_config(self):
+        """
+        predictor config setting
+        """
+        model_file_path = os.path.join(self.infer_model_path, 'model')
+        params_file_path = os.path.join(self.infer_model_path, 'params')
+
+        config = AnalysisConfig(model_file_path, params_file_path)
+        try:
+            _places = os.environ["CUDA_VISIBLE_DEVICES"]
+            int(_places[0])
+            use_gpu = True
+        except:
+            use_gpu = False
+
+        if use_gpu:
+            config.enable_use_gpu(8000, 0)
+        else:
+            config.disable_gpu()
+
+        config.disable_glog_info()
+
+        self.predictor = create_paddle_predictor(config)
+
+    def net(self, input_ids, position_ids, segment_ids, input_mask):
+        """
+        create neural network.
+        Args:
+            input_ids (tensor): the word ids.
+            position_ids (tensor): the position ids.
+            segment_ids (tensor): the segment ids.
+            input_mask (tensor): the padding mask.
+
+        Returns:
+            pooled_output (tensor):  sentence-level output for classification task.
+            sequence_output (tensor): token-level output for sequence task.
+        """
+        ernie = ErnieModel(
+            src_ids=input_ids,
+            position_ids=position_ids,
+            sentence_ids=segment_ids,
+            input_mask=input_mask,
+            config=self.ernie_config,
+            use_fp16=False)
+
+        pooled_output = ernie.get_pooled_output()
+        sequence_output = ernie.get_sequence_output()
+        return pooled_output, sequence_output
+
+    def array2tensor(self, arr_data):
+        """
+        convert numpy array to PaddleTensor
+        """
+        tensor_data = PaddleTensor(arr_data)
+        return tensor_data
+
+    @serving
+    def predict_sentiment(self, texts=[], use_gpu=False):
+        """
+        Get the sentiment label for the predicted texts. It will be classified as positive and negative.
+        Args:
+            texts (list(str)): the data to be predicted.
+            use_gpu (bool): Whether to use gpu or not.
+        Returns:
+            res (list): The result of sentiment label and probabilties.
+        """
+
+        if use_gpu:
+            try:
+                _places = os.environ["CUDA_VISIBLE_DEVICES"]
+                int(_places[0])
+            except:
+                raise RuntimeError(
+                    "Environment Variable CUDA_VISIBLE_DEVICES is not set correctly. If you wanna use gpu, please set CUDA_VISIBLE_DEVICES as cuda_device_id."
+                )
+
+        results = []
+        for text in texts:
+            feature = self._convert_text_to_feature(text)
+            inputs = [self.array2tensor(ndarray) for ndarray in feature]
+            output = self.predictor.run(inputs)
+            probilities = np.array(output[0].data.float_data())
+            label = self.label_map[np.argmax(probilities)]
+            result = {
+                'text': text,
+                'sentiment_label': label,
+                'positive_probs': probilities[1],
+                'negative_probs': probilities[0]
+            }
+            results.append(result)
+
+        return results
+
+    def _convert_text_to_feature(self, text):
+        """
+        Convert the raw text to feature which is needed to run program (feed_vars).
+        """
+        text_a = convert_to_unicode(text)
+        tokens_a = self.tokenizer.tokenize(text_a)
+        max_seq_len = 512
+
+        # Account for [CLS] and [SEP] with "- 2"
+        if len(tokens_a) > max_seq_len - 2:
+            tokens_a = tokens_a[0:(max_seq_len - 2)]
+
+        tokens = []
+        text_type_ids = []
+        tokens.append("[CLS]")
+        text_type_ids.append(0)
+        for token in tokens_a:
+            tokens.append(token)
+            text_type_ids.append(0)
+        tokens.append("[SEP]")
+        text_type_ids.append(0)
+
+        token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
+        position_ids = list(range(len(token_ids)))
+        task_ids = [0] * len(token_ids)
+
+        padded_token_ids, input_mask = pad_batch_data([token_ids],
+                                                      max_seq_len=max_seq_len,
+                                                      pad_idx=self.pad_id,
+                                                      return_input_mask=True)
+        padded_text_type_ids = pad_batch_data([text_type_ids],
+                                              max_seq_len=max_seq_len,
+                                              pad_idx=self.pad_id)
+        padded_position_ids = pad_batch_data([position_ids],
+                                             max_seq_len=max_seq_len,
+                                             pad_idx=self.pad_id)
+        padded_task_ids = pad_batch_data([task_ids],
+                                         max_seq_len=max_seq_len,
+                                         pad_idx=self.pad_id)
+
+        feature = [
+            padded_token_ids, padded_position_ids, padded_text_type_ids,
+            input_mask, padded_task_ids
+        ]
+        return feature
+
+    @runnable
+    def run_cmd(self, argvs):
+        """
+        Run as a command
+        """
+        self.parser = argparse.ArgumentParser(
+            description="Run the %s module." % self.name,
+            prog='hub run %s' % self.name,
+            usage='%(prog)s',
+            add_help=True)
+
+        self.arg_input_group = self.parser.add_argument_group(
+            title="Input options", description="Input data. Required")
+        self.arg_config_group = self.parser.add_argument_group(
+            title="Config options",
+            description=
+            "Run configuration for controlling module behavior, not required.")
+
+        self.add_module_config_arg()
+        self.add_module_input_arg()
+
+        args = self.parser.parse_args(argvs)
+        results = self.predict_sentiment(
+            texts=[args.input_text], use_gpu=args.use_gpu)
+        return results
+
+    def add_module_config_arg(self):
+        """
+        Add the command config options
+        """
+        self.arg_config_group.add_argument(
+            '--use_gpu',
+            type=ast.literal_eval,
+            default=False,
+            help="whether use GPU or not")
+
+    def add_module_input_arg(self):
+        """
+        Add the command input options
+        """
+        self.arg_input_group.add_argument(
+            '--input_text', type=str, default=None, help="data to be predicted")
+
+
+if __name__ == '__main__':
+    test_module = ErnieSkepSentimentAnalysis()
+    test_texts = ['你不是不聪明，而是不认真', '虽然小明很努力，但是他还是没有考100分']
+    results = test_module.predict_sentiment(test_texts, use_gpu=False)
+    print(results)
+    test_module.context(max_seq_len=128)
+    print(test_module.get_embedding(texts=[['你不是不聪明，而是不认真']]))
+    print(test_module.get_params_layer())

hub_module/scripts/configs/ernie_skep_sentiment_analysis.yml

+name: ernie_skep_sentiment_analysis
+dir: "modules/text/sentiment_analysis/ernie_skep_sentiment_analysis"
+exclude:
+  - README.md
+resources:
+  -
+    url: https://paddlehub.bj.bcebos.com/model/nlp/ernie_skep_sentiment_analysis/assets.tar.gz
+    dest: assets
+    uncompress: True

hub_module/tests/unittests/test_ernie_skep_sentiment_analysis.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from unittest import TestCase, main
+os.environ['CUDA_VISIBLE_DEVICES'] = '1'
+
+import numpy as np
+import paddlehub as hub
+
+
+class ErnieSkepSentimentAnalysisTestCase(TestCase):
+    def setUp(self):
+        self.module = hub.Module(name='ernie_skep_sentiment_analysis')
+        self.test_text = [[
+            '飞桨（PaddlePaddle）是国内开源产业级深度学习平台', 'PaddleHub是飞桨生态的预训练模型应用工具'
+        ], ["飞浆PaddleHub"]]
+        self.test_data = ['你不是不聪明，而是不认真', '虽然小明很努力，但是他还是没有考100分']
+        self.results = [{
+            'text': '你不是不聪明，而是不认真',
+            'sentiment_label': 'negative',
+            'positive_probs': 0.10738213360309601,
+            'negative_probs': 0.8926178216934204
+        },
+                        {
+                            'text': '虽然小明很努力，但是他还是没有考100分',
+                            'sentiment_label': 'negative',
+                            'positive_probs': 0.053915347903966904,
+                            'negative_probs': 0.9460846185684204
+                        }]
+
+    def test_predict_sentiment(self):
+        results_1 = self.module.predict_sentiment(self.test_data, use_gpu=False)
+        results_2 = self.module.predict_sentiment(self.test_data, use_gpu=True)
+
+        for index, res in enumerate(results_1):
+            self.assertEqual(res['text'], self.results[index]['text'])
+            self.assertEqual(res['sentiment_label'],
+                             self.results[index]['sentiment_label'])
+            self.assertTrue(
+                abs(res['positive_probs'] -
+                    self.results[index]['positive_probs']) < 1e-6)
+            self.assertTrue(
+                abs(res['negative_probs'] -
+                    self.results[index]['negative_probs']) < 1e-6)
+
+            self.assertEqual(res['text'], results_2[index]['text'])
+            self.assertEqual(res['sentiment_label'],
+                             results_2[index]['sentiment_label'])
+            self.assertTrue(
+                abs(res['positive_probs'] -
+                    results_2[index]['positive_probs']) < 1e-6)
+            self.assertTrue(
+                abs(res['negative_probs'] -
+                    results_2[index]['negative_probs']) < 1e-6)
+
+    def test_get_embedding(self):
+        # test batch_size
+        max_seq_len = 128
+        results = self.module.get_embedding(
+            texts=self.test_text,
+            use_gpu=False,
+            batch_size=1,
+            max_seq_len=max_seq_len)
+        results_2 = self.module.get_embedding(
+            texts=self.test_text,
+            use_gpu=False,
+            batch_size=10,
+            max_seq_len=max_seq_len)
+        # 2 sample results
+        self.assertEqual(len(results), 2)
+        self.assertEqual(len(results_2), 2)
+        # sequence embedding and token embedding results per sample
+        self.assertEqual(len(results[0]), 2)
+        self.assertEqual(len(results_2[0]), 2)
+        # sequence embedding shape
+        self.assertEqual(results[0][0].shape, (1024, ))
+        self.assertEqual(results_2[0][0].shape, (1024, ))
+        # token embedding shape
+        self.assertEqual(results[0][1].shape, (max_seq_len, 1024))
+        self.assertEqual(results_2[0][1].shape, (max_seq_len, 1024))
+
+        # test gpu
+        results_3 = self.module.get_embedding(
+            texts=self.test_text,
+            use_gpu=True,
+            batch_size=1,
+            max_seq_len=max_seq_len)
+        diff = np.abs(results[0][0] - results_3[0][0])
+        self.assertTrue((diff < 1e-6).all)
+        diff = np.abs(results[0][1] - results_3[0][1])
+        self.assertTrue((diff < 1e-6).all)
+        diff = np.abs(results[1][0] - results_3[1][0])
+        self.assertTrue((diff < 1e-6).all)
+        diff = np.abs(results[1][1] - results_3[1][1])
+        self.assertTrue((diff < 1e-6).all)
+
+    def test_get_params_layer(self):
+        self.module.context()
+        layers = self.module.get_params_layer()
+        layers = list(set(layers.values()))
+        true_layers = [i for i in range(24)]
+        self.assertEqual(layers, true_layers)
+
+    def test_get_spm_path(self):
+        self.assertEqual(self.module.get_spm_path(), None)
+
+    def test_get_word_dict_path(self):
+        self.assertEqual(self.module.get_word_dict_path(), None)
+
+    def test_get_vocab_path(self):
+        vocab_path = self.module.get_vocab_path()
+        true_vocab_path = os.path.join(self.module.directory, "assets",
+                                       "ernie_1.0_large_ch.vocab.txt")
+        self.assertEqual(vocab_path, true_vocab_path)
+
+
+if __name__ == '__main__':
+    main()

paddlehub/module/nlp_module.py

@@ -319,8 +319,6 @@ class TransformerModule(NLPBaseModule):
             pretraining_params_path,
             main_program=main_program,
             predicate=existed_params)
-        logger.info("Load pretraining parameters from {}.".format(
-            pretraining_params_path))
 
     def param_prefix(self):
         return "@HUB_%s@" % self.name