Merge branch 'develop' of https://github.com/PaddlePaddle/PaddleX into develop_qh

deploy/cpp/scripts/bootstrap.sh

+# download pre-compiled paddle encrypt
+ENCRYPTION_URL=https://bj.bcebos.com/paddlex/tools/paddlex-encryption.zip
+if [ ! -d "./paddlex-encryption" ]; then
+    wget -c ${ENCRYPTION_URL}
+    unzip paddlex-encryption.zip
+    rm -rf paddlex-encryption.zip
+fi
+
 # download pre-compiled opencv lib
 OPENCV_URL=https://paddleseg.bj.bcebos.com/deploy/docker/opencv3gcc4.8.tar.bz2
 if [ ! -d "./deps/opencv3gcc4.8" ]; then

deploy/cpp/scripts/build.sh

-# 是否使用GPU(即是否使用 CUDA)
-WITH_GPU=OFF
-# 使用MKL or openblas
-WITH_MKL=ON
-# 是否集成 TensorRT(仅WITH_GPU=ON 有效)
-WITH_TENSORRT=OFF
-# TensorRT 的lib路径
-TENSORRT_DIR=/path/to/TensorRT/
-# Paddle 预测库路径
-PADDLE_DIR=/path/to/fluid_inference/
-# Paddle 的预测库是否使用静态库来编译
-# 使用TensorRT时，Paddle的预测库通常为动态库
-WITH_STATIC_LIB=OFF
-# CUDA 的 lib 路径
-CUDA_LIB=/path/to/cuda/lib/
-# CUDNN 的 lib 路径
-CUDNN_LIB=/path/to/cudnn/lib/
-
-# 是否加载加密后的模型
-WITH_ENCRYPTION=OFF
-# 加密工具的路径
-ENCRYPTION_DIR=/path/to/encryption_tool/
-
-# OPENCV 路径, 如果使用自带预编译版本可不修改
-OPENCV_DIR=$(pwd)/deps/opencv3gcc4.8/
-sh $(pwd)/scripts/bootstrap.sh
-
-# 以下无需改动
-rm -rf build
-mkdir -p build
-cd build
-cmake .. \
-    -DWITH_GPU=${WITH_GPU} \
-    -DWITH_MKL=${WITH_MKL} \
-    -DWITH_TENSORRT=${WITH_TENSORRT} \
-    -DWITH_ENCRYPTION=${WITH_ENCRYPTION} \
-    -DTENSORRT_DIR=${TENSORRT_DIR} \
-    -DPADDLE_DIR=${PADDLE_DIR} \
-    -DWITH_STATIC_LIB=${WITH_STATIC_LIB} \
-    -DCUDA_LIB=${CUDA_LIB} \
-    -DCUDNN_LIB=${CUDNN_LIB} \
-    -DENCRYPTION_DIR=${ENCRYPTION_DIR} \
-    -DOPENCV_DIR=${OPENCV_DIR}
-make
+# 是否使用GPU(即是否使用 CUDA)
+WITH_GPU=OFF
+# 使用MKL or openblas
+WITH_MKL=ON
+# 是否集成 TensorRT(仅WITH_GPU=ON 有效)
+WITH_TENSORRT=OFF
+# TensorRT 的路径
+TENSORRT_DIR=/path/to/TensorRT/
+# Paddle 预测库路径
+PADDLE_DIR=/docker/jiangjiajun/PaddleDetection/deploy/cpp/fluid_inference
+# Paddle 的预测库是否使用静态库来编译
+# 使用TensorRT时，Paddle的预测库通常为动态库
+WITH_STATIC_LIB=OFF
+# CUDA 的 lib 路径
+CUDA_LIB=/usr/local/cuda/lib64
+# CUDNN 的 lib 路径
+CUDNN_LIB=/usr/local/cuda/lib64
+
+# 是否加载加密后的模型 
+WITH_ENCRYPTION=ON
+# 加密工具的路径, 如果使用自带预编译版本可不修改
+sh $(pwd)/scripts/bootstrap.sh # 下载预编译版本的加密工具
+ENCRYPTION_DIR=$(pwd)/paddlex-encryption
+
+# OPENCV 路径, 如果使用自带预编译版本可不修改
+sh $(pwd)/scripts/bootstrap.sh  # 下载预编译版本的opencv
+OPENCV_DIR=$(pwd)/deps/opencv3gcc4.8/
+
+# 以下无需改动
+rm -rf build
+mkdir -p build
+cd build
+cmake .. \
+    -DWITH_GPU=${WITH_GPU} \
+    -DWITH_MKL=${WITH_MKL} \
+    -DWITH_TENSORRT=${WITH_TENSORRT} \
+    -DWITH_ENCRYPTION=${WITH_ENCRYPTION} \
+    -DTENSORRT_DIR=${TENSORRT_DIR} \
+    -DPADDLE_DIR=${PADDLE_DIR} \
+    -DWITH_STATIC_LIB=${WITH_STATIC_LIB} \
+    -DCUDA_LIB=${CUDA_LIB} \
+    -DCUDNN_LIB=${CUDNN_LIB} \
+    -DENCRYPTION_DIR=${ENCRYPTION_DIR} \
+    -DOPENCV_DIR=${OPENCV_DIR}
+make

deploy/cpp/src/paddlex.cpp

@@ -31,6 +31,8 @@ void Model::create_predictor(const std::string& model_dir,
   std::string params_file = model_dir + OS_PATH_SEP + "__params__";
 #ifdef WITH_ENCRYPTION
   if (key != ""){
+    model_file = model_dir + OS_PATH_SEP + "__model__.encrypted";
+    params_file = model_dir + OS_PATH_SEP + "__params__.encrypted";
     paddle_security_load_model(&config, key.c_str(), model_file.c_str(), params_file.c_str());
   }
 #endif

docs/apis/deploy.md

-# paddlex.deploy
+# Predictor部署-paddlex.deploy
 
 使用AnalysisPredictor进行预测部署。
 
-## create_predictor
+## Predictor类
 
 ```
-paddlex.deploy.create_predictor(model_dir, use_gpu=False)
+paddlex.deploy.Predictor(model_dir, use_gpu=False, gpu_id=0, use_mkl=False, use_trt=False, use_glog=False, memory_optimize=True)
 ```
 
-#### Args
+> **参数**
 
-* **model_dir**: 训练过程中保存的模型路径
-* **use_gpu**: 是否使用GPU进行预测
+> > * **model_dir**: 训练过程中保存的模型路径, 注意需要使用导出的inference模型
+> > * **use_gpu**: 是否使用GPU进行预测
+> > * **gpu_id**: 使用的GPU序列号
+> > * **use_mkl**: 是否使用mkldnn加速库
+> > * **use_trt**: 是否使用TensorRT预测引擎
+> > * **use_glog**: 是否打印中间日志
+> > * **memory_optimize**: 是否优化内存使用
 
-#### Returns
+> > ### 示例
+> >
+> > ```
+> > import paddlex
+> > model = paddlex.deploy.Predictor(model_dir, use_gpu=True)
+> > result = model.predict(image_file)
+> > ```
 
-* **Predictor**: paddlex.deploy.predictor.Predictor
+### predict 接口
+> ```
+> predict(image, topk=1)
+> ```
 
-### 示例
+> **参数
 
-```
-import paddlex
-# 下
-Predictor = paddlex.deploy.create_predictor(model_dir, use_gpu=True)
-```
-
-## ClassifyPredictor
-继承至paddlex.deploy.predictor.Predictor，当model_dir/model.yml里面
-
-```
-paddlex.deploy.create_predictor(model_dir, use_gpu=False)
-```
-
-#### Args
-
-* **model_dir**: 训练过程中保存的模型路径
-* **use_gpu**: 是否使用GPU进行预测
-
-#### Returns
-
-* **Predictor**: paddlex.deploy.predictor.Predictor
-
-### 示例
-
-```
-import paddlex
-# 下
-Predictor = paddlex.deploy.create_predictor(model_dir, use_gpu=True)
-```
+* **image(str|np.ndarray)**: 待预测的图片路径或np.ndarray，若为后者需注意为BGR格式
+* **topk(int)**: 图像分类时使用的参数，表示预测前topk个可能的分类

docs/apis/index.rst

@@ -10,3 +10,4 @@ PaddleX API说明文档
    slim.md
    load_model.md
    visualize.md
+   deploy.md

docs/apis/slim.md

@@ -8,7 +8,7 @@ paddlex.slim.cal_params_sensitivities(model, save_file, eval_dataset, batch_size
 1. 获取模型中可裁剪卷积Kernel的名称。
 2. 计算每个可裁剪卷积Kernel不同裁剪率下的敏感度。
 【注意】卷积的敏感度是指在不同裁剪率下评估数据集预测精度的损失，通过得到的敏感度，可以决定最终模型需要裁剪的参数列表和各裁剪参数对应的裁剪率。  
-[查看使用示例](https://github.com/PaddlePaddle/PaddleX/blob/develop/tutorials/compress/classification/cal_sensitivities_file.py#L33)
+[查看使用示例](https://github.com/PaddlePaddle/PaddleX/blob/develop/tutorials/compress/classification/cal_sensitivities_file.py#L33) [查看裁剪教程](../tutorials/compress/classification.md)
 
 ### 参数
 

docs/tutorials/deploy/deploy_cpp_linux.md

@@ -67,6 +67,12 @@ CUDA_LIB=/path/to/cuda/lib/
 # CUDNN 的 lib 路径
 CUDNN_LIB=/path/to/cudnn/lib/
 
+# 是否加载加密后的模型
+WITH_ENCRYPTION=ON
+# 加密工具的路径, 如果使用自带预编译版本可不修改
+sh $(pwd)/scripts/bootstrap.sh # 下载预编译版本的加密工具
+ENCRYPTION_DIR=$(pwd)/paddlex-encryption
+
 # OPENCV 路径, 如果使用自带预编译版本可不修改
 OPENCV_DIR=$(pwd)/deps/opencv3gcc4.8/
 sh $(pwd)/scripts/bootstrap.sh
@@ -79,11 +85,13 @@ cmake .. \
     -DWITH_GPU=${WITH_GPU} \
     -DWITH_MKL=${WITH_MKL} \
     -DWITH_TENSORRT=${WITH_TENSORRT} \
+    -DWITH_ENCRYPTION=${WITH_ENCRYPTION} \
     -DTENSORRT_DIR=${TENSORRT_DIR} \
     -DPADDLE_DIR=${PADDLE_DIR} \
     -DWITH_STATIC_LIB=${WITH_STATIC_LIB} \
     -DCUDA_LIB=${CUDA_LIB} \
     -DCUDNN_LIB=${CUDNN_LIB} \
+    -DENCRYPTION_DIR=${ENCRYPTION_DIR} \
     -DOPENCV_DIR=${OPENCV_DIR}
 make
 

docs/tutorials/deploy/encryption.md

@@ -2,48 +2,76 @@
 
 飞桨团队推出模型加密方案，使用业内主流的AES加密技术对最终模型进行加密。飞桨用户可以通过PaddleX导出模型后，使用该方案对模型进行加密，预测时使用解密SDK进行模型解密并完成推理，大大提升AI应用安全性和开发效率。
 
-## 1. 方案介绍
+**注意：目前加密方案仅支持Linux系统**
 
-### 1.1 工具组成
+## 1. 方案简介
 
-[PaddleX模型加密SDK下载](https://bj.bcebos.com/paddlex/tools/paddlex-encryption.zip)
+### 1.1 加密工具
 
-下载并解压后，目录包含内容为：
+[PaddleX模型加密工具](https://bj.bcebos.com/paddlex/tools/paddlex-encryption.zip)。在编译部署代码时，编译脚本会自动下载加密工具，您也可以选择手动下载。
+
+加密工具包含内容为：
 ```
-paddle_model_encrypt
-├── include # 头文件：paddle_model_decrypt.h(解密)和paddle_model_encrypt.h(加密)
+paddlex-encryption
+├── include # 头文件：paddle_model_decrypt.h（解密）和paddle_model_encrypt.h（加密）
 |
 ├── lib # libpmodel-encrypt.so和libpmodel-decrypt.so动态库
 |
-└── tool # paddle_encrypt_tool
+└── tool # paddlex_encrypt_tool
 ```
 
-### 1.2 二进制工具
-
-#### 1.2.1 生成密钥
+### 1.2 加密PaddleX模型
 
-产生随机密钥信息（用于AES加解密使用）（32字节key + 16字节iv， 注意这里产生的key是经过base64编码后的，这样可以扩充选取key的范围）
+对模型完成加密后，加密工具会产生随机密钥信息(用于AES加解密使用），需要在后续加密部署时传入该密钥来用于解密。
+> 密钥由32字节key + 16字节iv组成， 注意这里产生的key是经过base64编码后的，这样可以扩充key的选取范围
 
 ```
-paddle_encrypt_tool    -g
+./paddlex-encryption/tool/paddlex_encrypt_tool -model_dir /path/to/paddlex_inference_model -save_dir /path/to/paddlex_encrypted_model
 ```
-#### 1.2.1 文件加密
 
+`-model_dir`用于指定inference模型路径，可使用[导出小度熊识别模型](deploy.md#导出inference模型)中导出的`inference_model`。加密完成后，加密过的模型会保存至指定的`-save_dir`下，包含`__model__.encrypted`、`__params__.encrypted`和`model.yml`三个文件，同时生成密钥信息，命令输出如下图所示，密钥为`kLAl1qOs5uRbFt0/RrIDTZW2+tOf5bzvUIaHGF8lJ1c=`
+![](images/encryt.png)
+
+## 2. PaddleX C++加密部署
+
+参考[Linux平台编译指南](deploy_cpp_linux.md)编译C++部署代码。编译成功后，预测demo的可执行程序分别为`build/demo/detector`，`build/demo/classifer`，`build/demo/segmenter`，用户可根据自己的模型类型选择，其主要命令参数说明如下：
+
+|  参数   | 说明  |
+|  ----  | ----  |
+| model_dir  | 导出的预测模型所在路径 |
+| image  | 要预测的图片文件路径 |
+| image_list  | 按行存储图片路径的.txt文件 |
+| use_gpu  | 是否使用 GPU 预测, 支持值为0或1(默认值为0) |
+| use_trt  | 是否使用 TensorTr 预测, 支持值为0或1(默认值为0) |
+| gpu_id  | GPU 设备ID, 默认值为0 |
+| save_dir | 保存可视化结果的路径, 默认值为"output"，classfier无该参数 |
+| key | 加密过程中产生的密钥信息，默认值为""表示加载的是未加密的模型 |
+
+
+## 样例
+
+可使用[导出小度熊识别模型](deploy.md#导出inference模型)中的测试图片进行预测。
+
+`样例一`：
+
+不使用`GPU`测试图片 `/path/to/xiaoduxiong.jpeg`  
+
+```shell
+./build/demo/detector --model_dir=/path/to/inference_model --image=/path/to/xiaoduxiong.jpeg --save_dir=output --key=kLAl1qOs5uRbFt0/RrIDTZW2+tOf5bzvUIaHGF8lJ1c=
 ```
- paddle_encrypt_tool    -e    -key    keydata     -infile    infile    -outfile    outfile
-```
+`--key`传入加密工具输出的密钥，例如`kLAl1qOs5uRbFt0/RrIDTZW2+tOf5bzvUIaHGF8lJ1c=`, 图片文件`可视化预测结果`会保存在`save_dir`参数设置的目录下。
 
-#### 1.3 SDK
 
+`样例二`:
+
+使用`GPU`预测多个图片`/path/to/image_list.txt`，image_list.txt内容的格式如下：
 ```
-// 加密API
-int paddle_encrypt_model(const char* keydata, const char* infile, const char* outfile);
-// 加载加密模型API：
-int paddle_security_load_model(
-        paddle::AnalysisConfig *config,
-        const char *key,
-        const char *model_file,
-        const char *param_file);
+/path/to/images/xiaoduxiong1.jpeg
+/path/to/images/xiaoduxiong2.jpeg
+...
+/path/to/images/xiaoduxiongn.jpeg
 ```
-
-## 2. PaddleX C++加密部署
+```shell
+./build/demo/detector --model_dir=/path/to/models/inference_model --image_list=/root/projects/images_list.txt --use_gpu=1 --save_dir=output --key=kLAl1qOs5uRbFt0/RrIDTZW2+tOf5bzvUIaHGF8lJ1c=
+```
+`--key`传入加密工具输出的密钥，例如`kLAl1qOs5uRbFt0/RrIDTZW2+tOf5bzvUIaHGF8lJ1c=`, 图片文件`可视化预测结果`会保存在`save_dir`参数设置的目录下。

docs/tutorials/deploy/index.rst

 多端部署
 ==============
+
+.. toctree::
+   :maxdepth: 2
+   :caption: 文档目录:
+
+   

paddlex/__init__.py

@@ -29,6 +29,8 @@ from . import cls
 from . import slim
 from . import convertor
 from . import tools
+from . import interpret
+from . import deploy
 
 try:
     import pycocotools
@@ -40,9 +42,9 @@ except:
         "[WARNING] pycocotools install: https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/install.md"
     )
 
-#import paddlehub as hub
-#if hub.version.hub_version < '1.6.2':
-#    raise Exception("[ERROR] paddlehub >= 1.6.2 is required")
+import paddlehub as hub
+if hub.version.hub_version < '1.6.2':
+    raise Exception("[ERROR] paddlehub >= 1.6.2 is required")
 
 env_info = get_environ_info()
 load_model = cv.models.load_model

paddlex/cv/models/base.py

@@ -375,6 +375,8 @@ class BaseAPI:
                    use_vdl=False,
                    early_stop=False,
                    early_stop_patience=5):
+        if train_dataset.num_samples < train_batch_size:
+            raise Exception('The amount of training datset must be larger than batch size.')
         if not osp.isdir(save_dir):
             if osp.exists(save_dir):
                 os.remove(save_dir)

paddlex/cv/models/classifier.py

@@ -27,7 +27,6 @@ from .base import BaseAPI
 
 class BaseClassifier(BaseAPI):
     """构建分类器，并实现其训练、评估、预测和模型导出。
-
     Args:
         model_name (str): 分类器的模型名字，取值范围为['ResNet18',
                           'ResNet34', 'ResNet50', 'ResNet101',
@@ -65,6 +64,8 @@ class BaseClassifier(BaseAPI):
         softmax_out = fluid.layers.softmax(net_out, use_cudnn=False)
         inputs = OrderedDict([('image', image)])
         outputs = OrderedDict([('predict', softmax_out)])
+        if mode == 'test':
+            self.interpretation_feats = OrderedDict([('logits', net_out)])
         if mode != 'test':
             cost = fluid.layers.cross_entropy(input=softmax_out, label=label)
             avg_cost = fluid.layers.mean(cost)
@@ -115,7 +116,6 @@ class BaseClassifier(BaseAPI):
               early_stop_patience=5,
               resume_checkpoint=None):
         """训练。
-
         Args:
             num_epochs (int): 训练迭代轮数。
             train_dataset (paddlex.datasets): 训练数据读取器。
@@ -139,7 +139,6 @@ class BaseClassifier(BaseAPI):
             early_stop_patience (int): 当使用提前终止训练策略时，如果验证集精度在`early_stop_patience`个epoch内
                 连续下降或持平，则终止训练。默认值为5。
             resume_checkpoint (str): 恢复训练时指定上次训练保存的模型路径。若为None，则不会恢复训练。默认值为None。
-
         Raises:
             ValueError: 模型从inference model进行加载。
         """
@@ -183,13 +182,11 @@ class BaseClassifier(BaseAPI):
                  epoch_id=None,
                  return_details=False):
         """评估。
-
         Args:
             eval_dataset (paddlex.datasets): 验证数据读取器。
             batch_size (int): 验证数据批大小。默认为1。
             epoch_id (int): 当前评估模型所在的训练轮数。
             return_details (bool): 是否返回详细信息。
-
         Returns:
           dict: 当return_details为False时，返回dict, 包含关键字：'acc1'、'acc5'，
               分别表示最大值的accuracy、前5个最大值的accuracy。
@@ -248,12 +245,10 @@ class BaseClassifier(BaseAPI):
 
     def predict(self, img_file, transforms=None, topk=1):
         """预测。
-
         Args:
             img_file (str): 预测图像路径。
             transforms (paddlex.cls.transforms): 数据预处理操作。
             topk (int): 预测时前k个最大值。
-
         Returns:
             list: 其中元素均为字典。字典的关键字为'category_id'、'category'、'score'，
             分别对应预测类别id、预测类别标签、预测得分。

paddlex/cv/models/slim/visualize.py

@@ -30,7 +30,6 @@ def visualize(model, sensitivities_file, save_dir='./'):
     import matplotlib
     matplotlib.use('Agg')
     import matplotlib.pyplot as plt
-
     program = model.test_prog
     place = model.places[0]
     fig = plt.figure()
@@ -51,15 +50,21 @@ def visualize(model, sensitivities_file, save_dir='./'):
         min(np.array(x)) - 0.01,
         max(np.array(x)) + 0.01, 0.05)
     my_y_ticks = np.arange(0.05, 1, 0.05)
-    plt.xticks(my_x_ticks, fontsize=3)
-    plt.yticks(my_y_ticks, fontsize=3)
+    plt.xticks(my_x_ticks, rotation=30, fontsize=8)
+    plt.yticks(my_y_ticks, fontsize=8)
     for a, b in zip(x, y):
         plt.text(
             a,
-            b, (float('%0.4f' % a), float('%0.3f' % b)),
+            b, (float('%0.3f' % a), float('%0.3f' % b)),
             ha='center',
             va='bottom',
-            fontsize=3)
+            fontsize=8)
+    plt.rcParams['savefig.dpi'] = 120
+    plt.rcParams['figure.dpi'] = 150
     suffix = osp.splitext(sensitivities_file)[-1]
-    plt.savefig('sensitivities.png', dpi=800)
+    plt.savefig(osp.join(save_dir, 'sensitivities.png'))
     plt.close()
+    import pickle
+    coor = dict(zip(x, y))
+    output = open(osp.join(save_dir, 'sensitivities_xy.pkl'), 'wb')
+    pickle.dump(coor, output)

paddlex/cv/nets/darknet.py

@@ -136,10 +136,8 @@ class DarkNet(object):
     def __call__(self, input):
         """
         Get the backbone of DarkNet, that is output for the 5 stages.
-
         Args:
             input (Variable): input variable.
-
         Returns:
             The last variables of each stage.
         """
@@ -184,4 +182,4 @@ class DarkNet(object):
                 bias_attr=ParamAttr(name='fc_offset'))
             return out
 
-        return blocks
+        return blocks
\ No newline at end of file

paddlex/cv/nets/densenet.py

@@ -173,4 +173,4 @@ class DenseNet(object):
             bn_ac_conv = fluid.layers.dropout(
                 x=bn_ac_conv, dropout_prob=dropout)
         bn_ac_conv = fluid.layers.concat([input, bn_ac_conv], axis=1)
-        return bn_ac_conv
+        return bn_ac_conv
\ No newline at end of file

paddlex/cv/nets/mobilenet_v1.py

@@ -24,7 +24,6 @@ from paddle.fluid.regularizer import L2Decay
 class MobileNetV1(object):
     """
     MobileNet v1, see https://arxiv.org/abs/1704.04861
-
     Args:
         norm_type (str): normalization type, 'bn' and 'sync_bn' are supported
         norm_decay (float): weight decay for normalization layer weights
@@ -214,4 +213,4 @@ class MobileNetV1(object):
         module17 = self._extra_block(module16, num_filters[3][0],
                                      num_filters[3][1], 1, 2,
                                      self.prefix_name + "conv7_4")
-        return module11, module13, module14, module15, module16, module17
+        return module11, module13, module14, module15, module16, module17
\ No newline at end of file

paddlex/cv/nets/mobilenet_v2.py

@@ -239,4 +239,4 @@ class MobileNetV2:
                 padding=1,
                 expansion_factor=t,
                 name=name + '_' + str(i + 1))
-        return last_residual_block, depthwise_output
+        return last_residual_block, depthwise_output
\ No newline at end of file

paddlex/cv/nets/shufflenet_v2.py

@@ -269,4 +269,4 @@ class ShuffleNetV2():
                 name='stage_' + name + '_conv3')
             out = fluid.layers.concat([conv_linear_1, conv_linear_2], axis=1)
 
-        return self.channel_shuffle(out, 2)
+        return self.channel_shuffle(out, 2)
\ No newline at end of file

paddlex/cv/nets/xception.py

@@ -329,4 +329,4 @@ def xception_41(num_classes=None):
 
 def xception_71(num_classes=None):
     model = Xception(num_classes, 71)
-    return model
+    return model
\ No newline at end of file

paddlex/cv/transforms/cls_transforms.py

@@ -86,7 +86,7 @@ class Compose(ClsTransform):
             else:
                 import imgaug.augmenters as iaa
                 if isinstance(op, iaa.Augmenter):
-                    im, = execute_imgaug(op, im)
+                    im = execute_imgaug(op, im)
                 outputs = (im, )
                 if label is not None:
                     outputs = (im, label)

paddlex/deploy.py

+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import os.path as osp
+import cv2
+import numpy as np
+import yaml
+import paddlex
+import paddle.fluid as fluid
+
+
+class Predictor:
+    def __init__(self,
+                 model_dir,
+                 use_gpu=True,
+                 gpu_id=0,
+                 use_mkl=False,
+                 use_trt=False,
+                 use_glog=False,
+                 memory_optimize=True):
+        """ 创建Paddle Predictor
+
+            Args:
+                model_dir: 模型路径（必须是导出的部署或量化模型）
+                use_gpu: 是否使用gpu，默认True
+                gpu_id: 使用gpu的id，默认0
+                use_mkl: 是否使用mkldnn计算库，CPU情况下使用，默认False
+                use_trt: 是否使用TensorRT，默认False
+                use_glog: 是否启用glog日志, 默认False
+                memory_optimize: 是否启动内存优化，默认True
+        """
+        if not osp.isdir(model_dir):
+            raise Exception("[ERROR] Path {} not exist.".format(model_dir))
+        if not osp.exists(osp.join(model_dir, "model.yml")):
+            raise Exception("There's not model.yml in {}".format(model_dir))
+        with open(osp.join(model_dir, "model.yml")) as f:
+            self.info = yaml.load(f.read(), Loader=yaml.Loader)
+
+        self.status = self.info['status']
+
+        if self.status != "Quant" and self.status != "Infer":
+            raise Exception("[ERROR] Only quantized model or exported "
+                            "inference model is supported.")
+
+        self.model_dir = model_dir
+        self.model_type = self.info['_Attributes']['model_type']
+        self.model_name = self.info['Model']
+        self.num_classes = self.info['_Attributes']['num_classes']
+        self.labels = self.info['_Attributes']['labels']
+        if self.info['Model'] == 'MaskRCNN':
+            if self.info['_init_params']['with_fpn']:
+                self.mask_head_resolution = 28
+            else:
+                self.mask_head_resolution = 14
+        transforms_mode = self.info.get('TransformsMode', 'RGB')
+        if transforms_mode == 'RGB':
+            to_rgb = True
+        else:
+            to_rgb = False
+        self.transforms = self.build_transforms(self.info['Transforms'],
+                                                to_rgb)
+        self.predictor = self.create_predictor(
+            use_gpu, gpu_id, use_mkl, use_trt, use_glog, memory_optimize)
+
+    def create_predictor(self,
+                         use_gpu=True,
+                         gpu_id=0,
+                         use_mkl=False,
+                         use_trt=False,
+                         use_glog=False,
+                         memory_optimize=True):
+        config = fluid.core.AnalysisConfig(
+            os.path.join(self.model_dir, '__model__'),
+            os.path.join(self.model_dir, '__params__'))
+
+        if use_gpu:
+            # 设置GPU初始显存(单位M)和Device ID
+            config.enable_use_gpu(100, gpu_id)
+        else:
+            config.disable_gpu()
+        if use_mkl:
+            config.enable_mkldnn()
+        if use_glog:
+            config.enable_glog_info()
+        else:
+            config.disable_glog_info()
+        if memory_optimize:
+            config.enable_memory_optim()
+        else:
+            config.diable_memory_optim()
+
+        # 开启计算图分析优化，包括OP融合等
+        config.switch_ir_optim(True)
+        # 关闭feed和fetch OP使用，使用ZeroCopy接口必须设置此项
+        config.switch_use_feed_fetch_ops(False)
+        predictor = fluid.core.create_paddle_predictor(config)
+        return predictor
+
+    def build_transforms(self, transforms_info, to_rgb=True):
+        if self.model_type == "classifier":
+            from paddlex.cls import transforms
+        elif self.model_type == "detector":
+            from paddlex.det import transforms
+        elif self.model_type == "segmenter":
+            from paddlex.seg import transforms
+        op_list = list()
+        for op_info in transforms_info:
+            op_name = list(op_info.keys())[0]
+            op_attr = op_info[op_name]
+            if not hasattr(transforms, op_name):
+                raise Exception(
+                    "There's no operator named '{}' in transforms of {}".
+                    format(op_name, self.model_type))
+            op_list.append(getattr(transforms, op_name)(**op_attr))
+        eval_transforms = transforms.Compose(op_list)
+        if hasattr(eval_transforms, 'to_rgb'):
+            eval_transforms.to_rgb = to_rgb
+        self.arrange_transforms(eval_transforms)
+        return eval_transforms
+
+    def arrange_transforms(self, transforms):
+        if self.model_type == 'classifier':
+            arrange_transform = paddlex.cls.transforms.ArrangeClassifier
+        elif self.model_type == 'segmenter':
+            arrange_transform = paddlex.seg.transforms.ArrangeSegmenter
+        elif self.model_type == 'detector':
+            arrange_name = 'Arrange{}'.format(self.model_name)
+            arrange_transform = getattr(paddlex.det.transforms, arrange_name)
+        else:
+            raise Exception("Unrecognized model type: {}".format(
+                self.model_type))
+        if type(transforms.transforms[-1]).__name__.startswith('Arrange'):
+            transforms.transforms[-1] = arrange_transform(mode='test')
+        else:
+            transforms.transforms.append(arrange_transform(mode='test'))
+
+    def preprocess(self, image):
+        """ 对图像做预处理
+
+            Args:
+                image(str|np.ndarray): 图片路径或np.ndarray，如为后者，要求是BGR格式
+        """
+        res = dict()
+        if self.model_type == "classifier":
+            im, = self.transforms(image)
+            im = np.expand_dims(im, axis=0).copy()
+            res['image'] = im
+        elif self.model_type == "detector":
+            if self.model_name == "YOLOv3":
+                im, im_shape = self.transforms(image)
+                im = np.expand_dims(im, axis=0).copy()
+                im_shape = np.expand_dims(im_shape, axis=0).copy()
+                res['image'] = im
+                res['im_size'] = im_shape
+            if self.model_name.count('RCNN') > 0:
+                im, im_resize_info, im_shape = self.transforms(image)
+                im = np.expand_dims(im, axis=0).copy()
+                im_resize_info = np.expand_dims(im_resize_info, axis=0).copy()
+                im_shape = np.expand_dims(im_shape, axis=0).copy()
+                res['image'] = im
+                res['im_info'] = im_resize_info
+                res['im_shape'] = im_shape
+        elif self.model_type == "segmenter":
+            im, im_info = self.transforms(image)
+            im = np.expand_dims(im, axis=0).copy()
+            res['image'] = im
+            res['im_info'] = im_info
+        return res
+
+    def raw_predict(self, inputs):
+        """ 接受预处理过后的数据进行预测
+
+            Args:
+                inputs(tuple): 预处理过后的数据
+        """
+        for k, v in inputs.items():
+            try:
+                tensor = self.predictor.get_input_tensor(k)
+            except:
+                continue
+            tensor.copy_from_cpu(v)
+        self.predictor.zero_copy_run()
+        output_names = self.predictor.get_output_names()
+        output_results = list()
+        for name in output_names:
+            output_tensor = self.predictor.get_output_tensor(name)
+            output_results.append(output_tensor.copy_to_cpu())
+        return output_results
+
+    def classifier_postprocess(self, preds, topk=1):
+        """ 对分类模型的预测结果做后处理
+        """
+        true_topk = min(self.num_classes, topk)
+        pred_label = np.argsort(preds[0][0])[::-1][:true_topk]
+        result = [{
+            'category_id': l,
+            'category': self.labels[l],
+            'score': preds[0][0, l],
+        } for l in pred_label]
+        return result
+
+    def segmenter_postprocess(self, preds, preprocessed_inputs):
+        """ 对语义分割结果做后处理
+        """
+        label_map = np.squeeze(preds[0]).astype('uint8')
+        score_map = np.squeeze(preds[1])
+        score_map = np.transpose(score_map, (1, 2, 0))
+        im_info = preprocessed_inputs['im_info']
+        for info in im_info[::-1]:
+            if info[0] == 'resize':
+                w, h = info[1][1], info[1][0]
+                label_map = cv2.resize(label_map, (w, h), cv2.INTER_NEAREST)
+                score_map = cv2.resize(score_map, (w, h), cv2.INTER_LINEAR)
+            elif info[0] == 'padding':
+                w, h = info[1][1], info[1][0]
+                label_map = label_map[0:h, 0:w]
+                score_map = score_map[0:h, 0:w, :]
+            else:
+                raise Exception("Unexpected info '{}' in im_info".format(info[
+                    0]))
+        return {'label_map': label_map, 'score_map': score_map}
+
+    def detector_postprocess(self, preds, preprocessed_inputs):
+        """ 对目标检测和实例分割结果做后处理
+        """
+        bboxes = {'bbox': (np.array(preds[0]), [[len(preds[0])]])}
+        bboxes['im_id'] = (np.array([[0]]).astype('int32'), [])
+        clsid2catid = dict({i: i for i in range(self.num_classes)})
+        xywh_results = paddlex.cv.models.utils.detection_eval.bbox2out(
+            [bboxes], clsid2catid)
+        results = list()
+        for xywh_res in xywh_results:
+            del xywh_res['image_id']
+            xywh_res['category'] = self.labels[xywh_res['category_id']]
+            results.append(xywh_res)
+        if len(preds) > 1:
+            im_shape = preprocessed_inputs['im_shape']
+            bboxes['im_shape'] = (im_shape, [])
+            bboxes['mask'] = (np.array(preds[1]), [[len(preds[1])]])
+            segm_results = paddlex.cv.models.utils.detection_eval.mask2out(
+                [bboxes], clsid2catid, self.mask_head_resolution)
+            import pycocotools.mask as mask_util
+            for i in range(len(results)):
+                results[i]['mask'] = mask_util.decode(segm_results[i][
+                    'segmentation'])
+        return results
+
+    def predict(self, image, topk=1, threshold=0.5):
+        """ 图片预测
+
+            Args:
+                image(str|np.ndarray): 图片路径或np.ndarray格式，如果后者，要求为BGR输入格式
+                topk(int): 分类预测时使用，表示预测前topk的结果
+        """
+        preprocessed_input = self.preprocess(image)
+        model_pred = self.raw_predict(preprocessed_input)
+
+        if self.model_type == "classifier":
+            results = self.classifier_postprocess(model_pred, topk)
+        elif self.model_type == "detector":
+            results = self.detector_postprocess(model_pred, preprocessed_input)
+        elif self.model_type == "segmenter":
+            results = self.segmenter_postprocess(model_pred,
+                                                 preprocessed_input)
+        return results

paddlex/interpret/__init__.py

+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from . import visualize
+
+visualize = visualize.visualize
\ No newline at end of file

paddlex/interpret/as_data_reader/__init__.py

+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

paddlex/interpret/as_data_reader/data_path_utils.py

+#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+import os
+
+def _find_classes(dir):
+    # Faster and available in Python 3.5 and above
+    classes = [d.name for d in os.scandir(dir) if d.is_dir()]
+    classes.sort()
+    class_to_idx = {classes[i]: i for i in range(len(classes))}
+    return classes, class_to_idx
\ No newline at end of file

paddlex/interpret/as_data_reader/readers.py

+#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+import os
+import sys
+import cv2
+import numpy as np
+import six
+import glob
+from .data_path_utils import _find_classes
+from PIL import Image
+
+
+def resize_short(img, target_size, interpolation=None):
+    """resize image
+
+    Args:
+        img: image data
+        target_size: resize short target size
+        interpolation: interpolation mode
+
+    Returns:
+        resized image data
+    """
+    percent = float(target_size) / min(img.shape[0], img.shape[1])
+    resized_width = int(round(img.shape[1] * percent))
+    resized_height = int(round(img.shape[0] * percent))
+    if interpolation:
+        resized = cv2.resize(
+            img, (resized_width, resized_height), interpolation=interpolation)
+    else:
+        resized = cv2.resize(img, (resized_width, resized_height))
+    return resized
+
+
+def crop_image(img, target_size, center=True):
+    """crop image
+
+    Args:
+        img: images data
+        target_size: crop target size
+        center: crop mode
+
+    Returns:
+        img: cropped image data
+    """
+    height, width = img.shape[:2]
+    size = target_size
+    if center:
+        w_start = (width - size) // 2
+        h_start = (height - size) // 2
+    else:
+        w_start = np.random.randint(0, width - size + 1)
+        h_start = np.random.randint(0, height - size + 1)
+    w_end = w_start + size
+    h_end = h_start + size
+    img = img[h_start:h_end, w_start:w_end, :]
+    return img
+
+
+def preprocess_image(img, random_mirror=False):
+    """
+    centered, scaled by 1/255.
+    :param img: np.array: shape: [ns, h, w, 3], color order: rgb.
+    :return: np.array: shape: [ns, h, w, 3]
+    """
+    mean = [0.485, 0.456, 0.406]
+    std = [0.229, 0.224, 0.225]
+
+    # transpose to [ns, 3, h, w]
+    img = img.astype('float32').transpose((0, 3, 1, 2)) / 255
+
+    img_mean = np.array(mean).reshape((3, 1, 1))
+    img_std = np.array(std).reshape((3, 1, 1))
+    img -= img_mean
+    img /= img_std
+
+    if random_mirror:
+        mirror = int(np.random.uniform(0, 2))
+        if mirror == 1:
+            img = img[:, :, ::-1, :]
+
+    return img
+
+
+def read_image(img_path, target_size=256, crop_size=224):
+    """
+    resize_short to 256, then center crop to 224.
+    :param img_path: one image path
+    :return: np.array: shape: [1, h, w, 3], color order: rgb.
+    """
+
+    if isinstance(img_path, str):
+        with open(img_path, 'rb') as f:
+            img = Image.open(f)
+            img = img.convert('RGB')
+            img = np.array(img)
+            # img = cv2.imread(img_path)
+
+            img = resize_short(img, target_size, interpolation=None)
+            img = crop_image(img, target_size=crop_size, center=True)
+            # img = img[:, :, ::-1]
+            img = np.expand_dims(img, axis=0)
+            return img
+    elif isinstance(img_path, np.ndarray):
+        assert len(img_path.shape) == 4
+        return img_path
+    else:
+        ValueError(f"Not recognized data type {type(img_path)}.")
+
+
+class ReaderConfig(object):
+    """
+    A generic data loader where the images are arranged in this way:
+
+        root/train/dog/xxy.jpg
+        root/train/dog/xxz.jpg
+        ...
+        root/train/cat/nsdf3.jpg
+        root/train/cat/asd932_.jpg
+        ...
+
+        root/test/dog/xxx.jpg
+        ...
+        root/test/cat/123.jpg
+        ...
+
+    """
+    def __init__(self, dataset_dir, is_test):
+        image_paths, labels, self.num_classes = self.get_dataset_info(dataset_dir, is_test)
+        random_per = np.random.permutation(range(len(image_paths)))
+        self.image_paths = image_paths[random_per]
+        self.labels = labels[random_per]
+        self.is_test = is_test
+
+    def get_reader(self):
+        def reader():
+            IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
+            target_size = 256
+            crop_size = 224
+
+            for i, img_path in enumerate(self.image_paths):
+                if not img_path.lower().endswith(IMG_EXTENSIONS):
+                    continue
+
+                img = cv2.imread(img_path)
+                if img is None:
+                    print(img_path)
+                    continue
+                img = resize_short(img, target_size, interpolation=None)
+                img = crop_image(img, crop_size, center=self.is_test)
+                img = img[:, :, ::-1]
+                img = np.expand_dims(img, axis=0)
+
+                img = preprocess_image(img, not self.is_test)
+
+                yield img, self.labels[i]
+
+        return reader
+
+    def get_dataset_info(self, dataset_dir, is_test=False):
+        IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
+
+        # read
+        if is_test:
+            datasubset_dir = os.path.join(dataset_dir, 'test')
+        else:
+            datasubset_dir = os.path.join(dataset_dir, 'train')
+
+        class_names, class_to_idx = _find_classes(datasubset_dir)
+        # num_classes = len(class_names)
+        image_paths = []
+        labels = []
+        for class_name in class_names:
+            classes_dir = os.path.join(datasubset_dir, class_name)
+            for img_path in glob.glob(os.path.join(classes_dir, '*')):
+                if not img_path.lower().endswith(IMG_EXTENSIONS):
+                    continue
+
+                image_paths.append(img_path)
+                labels.append(class_to_idx[class_name])
+
+        image_paths = np.array(image_paths)
+        labels = np.array(labels)
+        return image_paths, labels, len(class_names)
+
+
+def create_reader(list_image_path, list_label=None, is_test=False):
+    def reader():
+        IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
+        target_size = 256
+        crop_size = 224
+
+        for i, img_path in enumerate(list_image_path):
+            if not img_path.lower().endswith(IMG_EXTENSIONS):
+                continue
+
+            img = cv2.imread(img_path)
+            if img is None:
+                print(img_path)
+                continue
+
+            img = resize_short(img, target_size, interpolation=None)
+            img = crop_image(img, crop_size, center=is_test)
+            img = img[:, :, ::-1]
+            img_show = np.expand_dims(img, axis=0)
+
+            img = preprocess_image(img_show, not is_test)
+
+            label = 0 if list_label is None else list_label[i]
+
+            yield img_show, img, label
+
+    return reader
\ No newline at end of file

paddlex/interpret/core/__init__.py

+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

paddlex/interpret/core/_session_preparation.py

+#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+import os
+import paddle.fluid as fluid
+import numpy as np
+from paddle.fluid.param_attr import ParamAttr
+from ..as_data_reader.readers import preprocess_image
+
+root_path = os.environ['HOME']
+root_path = os.path.join(root_path, '.paddlex')
+h_pre_models = os.path.join(root_path, "pre_models")
+h_pre_models_kmeans = os.path.join(h_pre_models, "kmeans_model.pkl")
+
+
+def paddle_get_fc_weights(var_name="fc_0.w_0"):
+    fc_weights = fluid.global_scope().find_var(var_name).get_tensor()
+    return np.array(fc_weights)
+
+
+def paddle_resize(extracted_features, outsize):
+    resized_features = fluid.layers.resize_bilinear(extracted_features, outsize)
+    return resized_features
+
+
+def compute_features_for_kmeans(data_content):
+    def conv_bn_layer(input,
+                      num_filters,
+                      filter_size,
+                      stride=1,
+                      groups=1,
+                      act=None,
+                      name=None,
+                      is_test=True,
+                      global_name=''):
+        conv = fluid.layers.conv2d(
+            input=input,
+            num_filters=num_filters,
+            filter_size=filter_size,
+            stride=stride,
+            padding=(filter_size - 1) // 2,
+            groups=groups,
+            act=None,
+            param_attr=ParamAttr(name=global_name + name + "_weights"),
+            bias_attr=False,
+            name=global_name + name + '.conv2d.output.1')
+        if name == "conv1":
+            bn_name = "bn_" + name
+        else:
+            bn_name = "bn" + name[3:]
+        return fluid.layers.batch_norm(
+            input=conv,
+            act=act,
+            name=global_name + bn_name + '.output.1',
+            param_attr=ParamAttr(global_name + bn_name + '_scale'),
+            bias_attr=ParamAttr(global_name + bn_name + '_offset'),
+            moving_mean_name=global_name + bn_name + '_mean',
+            moving_variance_name=global_name + bn_name + '_variance',
+            use_global_stats=is_test
+        )
+
+    startup_prog = fluid.default_startup_program().clone(for_test=True)
+    prog = fluid.Program()
+    with fluid.program_guard(prog, startup_prog):
+        with fluid.unique_name.guard():
+            image_op = fluid.data(name='image', shape=[None, 3, 224, 224], dtype='float32')
+
+            conv = conv_bn_layer(
+                input=image_op,
+                num_filters=32,
+                filter_size=3,
+                stride=2,
+                act='relu',
+                name='conv1_1')
+            conv = conv_bn_layer(
+                input=conv,
+                num_filters=32,
+                filter_size=3,
+                stride=1,
+                act='relu',
+                name='conv1_2')
+            conv = conv_bn_layer(
+                input=conv,
+                num_filters=64,
+                filter_size=3,
+                stride=1,
+                act='relu',
+                name='conv1_3')
+            extracted_features = conv
+            resized_features = fluid.layers.resize_bilinear(extracted_features, image_op.shape[2:])
+
+    gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
+    place = fluid.CUDAPlace(gpu_id)
+    # place = fluid.CPUPlace()
+    exe = fluid.Executor(place)
+    exe.run(startup_prog)
+    fluid.io.load_persistables(exe, h_pre_models, prog)
+
+    images = preprocess_image(data_content)  # transpose to [N, 3, H, W], scaled to [0.0, 1.0]
+    result = exe.run(prog, fetch_list=[resized_features], feed={'image': images})
+
+    return result[0][0]
\ No newline at end of file

paddlex/interpret/core/interpretation.py

+#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+from .interpretation_algorithms import CAM, LIME, NormLIME
+from .normlime_base import precompute_normlime_weights
+
+
+class Interpretation(object):
+    """
+    Base class for all interpretation algorithms.
+    """
+    def __init__(self, interpretation_algorithm_name, predict_fn, label_names, **kwargs):
+        supported_algorithms = {
+            'cam': CAM,
+            'lime': LIME,
+            'normlime': NormLIME
+        }
+
+        self.algorithm_name = interpretation_algorithm_name.lower()
+        assert self.algorithm_name in supported_algorithms.keys()
+        self.predict_fn = predict_fn
+
+        # initialization for the interpretation algorithm.
+        self.algorithm = supported_algorithms[self.algorithm_name](
+            self.predict_fn, label_names, **kwargs
+        )
+
+    def interpret(self, data_, visualization=True, save_to_disk=True, save_dir='./tmp'):
+        """
+
+        Args:
+            data_: data_ can be a path or numpy.ndarray.
+            visualization: whether to show using matplotlib.
+            save_to_disk: whether to save the figure in local disk.
+            save_dir: dir to save figure if save_to_disk is True.
+
+        Returns:
+
+        """
+        return self.algorithm.interpret(data_, visualization, save_to_disk, save_dir)

paddlex/interpret/core/interpretation_algorithms.py

+#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+import os
+import numpy as np
+import time
+
+from . import lime_base
+from ..as_data_reader.readers import read_image
+from ._session_preparation import paddle_get_fc_weights, compute_features_for_kmeans, h_pre_models_kmeans
+from .normlime_base import combine_normlime_and_lime, get_feature_for_kmeans, load_kmeans_model
+
+import cv2
+
+
+class CAM(object):
+    def __init__(self, predict_fn, label_names):
+        """
+
+        Args:
+            predict_fn: input: images_show [N, H, W, 3], RGB range(0, 255)
+                        output: [
+                        logits [N, num_classes],
+                        feature map before global average pooling [N, num_channels, h_, w_]
+                        ]
+
+        """
+        self.predict_fn = predict_fn
+        self.label_names = label_names
+
+    def preparation_cam(self, data_):
+        image_show = read_image(data_)
+        result = self.predict_fn(image_show)
+
+        logit = result[0][0]
+        if abs(np.sum(logit) - 1.0) > 1e-4:
+            # softmax
+            logit = logit - np.max(logit)
+            exp_result = np.exp(logit)
+            probability = exp_result / np.sum(exp_result)
+        else:
+            probability = logit
+
+        # only interpret top 1
+        pred_label = np.argsort(probability)
+        pred_label = pred_label[-1:]
+
+        self.predicted_label = pred_label[0]
+        self.predicted_probability = probability[pred_label[0]]
+        self.image = image_show[0]
+        self.labels = pred_label
+
+        fc_weights = paddle_get_fc_weights()
+        feature_maps = result[1]
+        
+        l = pred_label[0]
+        ln = l
+        if self.label_names is not None:
+            ln = self.label_names[l]
+
+        print(f'predicted result: {ln} with probability {probability[pred_label[0]]:.3f}')
+        return feature_maps, fc_weights
+
+    def interpret(self, data_, visualization=True, save_to_disk=True, save_outdir=None):
+        feature_maps, fc_weights = self.preparation_cam(data_)
+        cam = get_cam(self.image, feature_maps, fc_weights, self.predicted_label)
+
+        if visualization or save_to_disk:
+            import matplotlib.pyplot as plt
+            from skimage.segmentation import mark_boundaries
+            l = self.labels[0]
+            ln = l 
+            if self.label_names is not None:
+                ln = self.label_names[l]
+
+            psize = 5
+            nrows = 1
+            ncols = 2
+
+            plt.close()
+            f, axes = plt.subplots(nrows, ncols, figsize=(psize * ncols, psize * nrows))
+            for ax in axes.ravel():
+                ax.axis("off")
+            axes = axes.ravel()
+            axes[0].imshow(self.image)
+            axes[0].set_title(f"label {ln}, proba: {self.predicted_probability: .3f}")
+
+            axes[1].imshow(cam)
+            axes[1].set_title("CAM")
+
+        if save_to_disk and save_outdir is not None:
+            os.makedirs(save_outdir, exist_ok=True)
+            save_fig(data_, save_outdir, 'cam')
+
+        if visualization:
+            plt.show()
+
+        return
+
+
+class LIME(object):
+    def __init__(self, predict_fn, label_names, num_samples=3000, batch_size=50):
+        """
+        LIME wrapper. See lime_base.py for the detailed LIME implementation.
+        Args:
+            predict_fn: from image [N, H, W, 3] to logits [N, num_classes], this is necessary for computing LIME.
+            num_samples: the number of samples that LIME takes for fitting.
+            batch_size: batch size for model inference each time.
+        """
+        self.num_samples = num_samples
+        self.batch_size = batch_size
+
+        self.predict_fn = predict_fn
+        self.labels = None
+        self.image = None
+        self.lime_interpreter = None
+        self.label_names = label_names
+
+    def preparation_lime(self, data_):
+        image_show = read_image(data_)
+        result = self.predict_fn(image_show)
+
+        result = result[0]  # only one image here.
+
+        if abs(np.sum(result) - 1.0) > 1e-4:
+            # softmax
+            result = result - np.max(result)
+            exp_result = np.exp(result)
+            probability = exp_result / np.sum(exp_result)
+        else:
+            probability = result
+
+        # only interpret top 1
+        pred_label = np.argsort(probability)
+        pred_label = pred_label[-1:]
+
+        self.predicted_label = pred_label[0]
+        self.predicted_probability = probability[pred_label[0]]
+        self.image = image_show[0]
+        self.labels = pred_label
+        
+        l = pred_label[0]
+        ln = l
+        if self.label_names is not None:
+            ln = self.label_names[l]
+            
+        print(f'predicted result: {ln} with probability {probability[pred_label[0]]:.3f}')
+
+        end = time.time()
+        algo = lime_base.LimeImageInterpreter()
+        interpreter = algo.interpret_instance(self.image, self.predict_fn, self.labels, 0,
+                                              num_samples=self.num_samples, batch_size=self.batch_size)
+        self.lime_interpreter = interpreter
+        print('lime time: ', time.time() - end, 's.')
+
+    def interpret(self, data_, visualization=True, save_to_disk=True, save_outdir=None):
+        if self.lime_interpreter is None:
+            self.preparation_lime(data_)
+
+        if visualization or save_to_disk:
+            import matplotlib.pyplot as plt
+            from skimage.segmentation import mark_boundaries
+            l = self.labels[0]
+            ln = l 
+            if self.label_names is not None:
+                ln = self.label_names[l]
+
+            psize = 5
+            nrows = 2
+            weights_choices = [0.6, 0.7, 0.75, 0.8, 0.85]
+            ncols = len(weights_choices)
+
+            plt.close()
+            f, axes = plt.subplots(nrows, ncols, figsize=(psize * ncols, psize * nrows))
+            for ax in axes.ravel():
+                ax.axis("off")
+            axes = axes.ravel()
+            axes[0].imshow(self.image)
+            axes[0].set_title(f"label {ln}, proba: {self.predicted_probability: .3f}")
+
+            axes[1].imshow(mark_boundaries(self.image, self.lime_interpreter.segments))
+            axes[1].set_title("superpixel segmentation")
+
+            # LIME visualization
+            for i, w in enumerate(weights_choices):
+                num_to_show = auto_choose_num_features_to_show(self.lime_interpreter, l, w)
+                temp, mask = self.lime_interpreter.get_image_and_mask(
+                    l, positive_only=False, hide_rest=False, num_features=num_to_show
+                )
+                axes[ncols + i].imshow(mark_boundaries(temp, mask))
+                axes[ncols + i].set_title(f"label {ln}, first {num_to_show} superpixels")
+
+        if save_to_disk and save_outdir is not None:
+            os.makedirs(save_outdir, exist_ok=True)
+            save_fig(data_, save_outdir, 'lime', self.num_samples)
+
+        if visualization:
+            plt.show()
+
+        return
+
+
+class NormLIME(object):
+    def __init__(self, predict_fn, label_names, num_samples=3000, batch_size=50,
+                 kmeans_model_for_normlime=None, normlime_weights=None):
+        if kmeans_model_for_normlime is None:
+            try:
+                self.kmeans_model = load_kmeans_model(h_pre_models_kmeans)
+            except:
+                raise ValueError("NormLIME needs the KMeans model, where we provided a default one in "
+                                 "pre_models/kmeans_model.pkl.")
+        else:
+            print("Warning: It is *strongly* suggested to use the default KMeans model in pre_models/kmeans_model.pkl. "
+                  "Use another one will change the final result.")
+            self.kmeans_model = load_kmeans_model(kmeans_model_for_normlime)
+
+        self.num_samples = num_samples
+        self.batch_size = batch_size
+
+        try:
+            self.normlime_weights = np.load(normlime_weights, allow_pickle=True).item()
+        except:
+            self.normlime_weights = None
+            print("Warning: not find the correct precomputed Normlime result.")
+
+        self.predict_fn = predict_fn
+
+        self.labels = None
+        self.image = None
+        self.label_names = label_names
+
+    def predict_cluster_labels(self, feature_map, segments):
+        return self.kmeans_model.predict(get_feature_for_kmeans(feature_map, segments))
+
+    def predict_using_normlime_weights(self, pred_labels, predicted_cluster_labels):
+        # global weights
+        g_weights = {y: [] for y in pred_labels}
+        for y in pred_labels:
+            cluster_weights_y = self.normlime_weights.get(y, {})
+            g_weights[y] = [
+                (i, cluster_weights_y.get(k, 0.0)) for i, k in enumerate(predicted_cluster_labels)
+            ]
+
+            g_weights[y] = sorted(g_weights[y],
+                                  key=lambda x: np.abs(x[1]), reverse=True)
+
+        return g_weights
+
+    def preparation_normlime(self, data_):
+        self._lime = LIME(
+            self.predict_fn,
+            self.label_names,
+            self.num_samples,
+            self.batch_size
+        )
+        self._lime.preparation_lime(data_)
+
+        image_show = read_image(data_)
+
+        self.predicted_label = self._lime.predicted_label
+        self.predicted_probability = self._lime.predicted_probability
+        self.image = image_show[0]
+        self.labels = self._lime.labels
+        # print(f'predicted result: {self.predicted_label} with probability {self.predicted_probability: .3f}')
+        print('performing NormLIME operations ...')
+
+        cluster_labels = self.predict_cluster_labels(
+            compute_features_for_kmeans(image_show).transpose((1, 2, 0)), self._lime.lime_interpreter.segments
+        )
+
+        g_weights = self.predict_using_normlime_weights(self.labels, cluster_labels)
+
+        return g_weights
+
+    def interpret(self, data_, visualization=True, save_to_disk=True, save_outdir=None):
+        if self.normlime_weights is None:
+            raise ValueError("Not find the correct precomputed NormLIME result. \n"
+                             "\t Try to call compute_normlime_weights() first or load the correct path.")
+
+        g_weights = self.preparation_normlime(data_)
+        lime_weights = self._lime.lime_interpreter.local_weights
+
+        if visualization or save_to_disk:
+            import matplotlib.pyplot as plt
+            from skimage.segmentation import mark_boundaries
+            l = self.labels[0]
+            ln = l
+            if self.label_names is not None:
+                ln = self.label_names[l]
+
+            psize = 5
+            nrows = 4
+            weights_choices = [0.6, 0.7, 0.75, 0.8, 0.85]
+            nums_to_show = []
+            ncols = len(weights_choices)
+
+            plt.close()
+            f, axes = plt.subplots(nrows, ncols, figsize=(psize * ncols, psize * nrows))
+            for ax in axes.ravel():
+                ax.axis("off")
+
+            axes = axes.ravel()
+            axes[0].imshow(self.image)
+            axes[0].set_title(f"label {ln}, proba: {self.predicted_probability: .3f}")
+
+            axes[1].imshow(mark_boundaries(self.image, self._lime.lime_interpreter.segments))
+            axes[1].set_title("superpixel segmentation")
+
+            # LIME visualization
+            for i, w in enumerate(weights_choices):
+                num_to_show = auto_choose_num_features_to_show(self._lime.lime_interpreter, l, w)
+                nums_to_show.append(num_to_show)
+                temp, mask = self._lime.lime_interpreter.get_image_and_mask(
+                    l, positive_only=False, hide_rest=False, num_features=num_to_show
+                )
+                axes[ncols + i].imshow(mark_boundaries(temp, mask))
+                axes[ncols + i].set_title(f"LIME: first {num_to_show} superpixels")
+
+            # NormLIME visualization
+            self._lime.lime_interpreter.local_weights = g_weights
+            for i, num_to_show in enumerate(nums_to_show):
+                temp, mask = self._lime.lime_interpreter.get_image_and_mask(
+                    l, positive_only=False, hide_rest=False, num_features=num_to_show
+                )
+                axes[ncols * 2 + i].imshow(mark_boundaries(temp, mask))
+                axes[ncols * 2 + i].set_title(f"NormLIME: first {num_to_show} superpixels")
+
+            # NormLIME*LIME visualization
+            combined_weights = combine_normlime_and_lime(lime_weights, g_weights)
+            self._lime.lime_interpreter.local_weights = combined_weights
+            for i, num_to_show in enumerate(nums_to_show):
+                temp, mask = self._lime.lime_interpreter.get_image_and_mask(
+                    l, positive_only=False, hide_rest=False, num_features=num_to_show
+                )
+                axes[ncols * 3 + i].imshow(mark_boundaries(temp, mask))
+                axes[ncols * 3 + i].set_title(f"Combined: first {num_to_show} superpixels")
+
+            self._lime.lime_interpreter.local_weights = lime_weights
+
+        if save_to_disk and save_outdir is not None:
+            os.makedirs(save_outdir, exist_ok=True)
+            save_fig(data_, save_outdir, 'normlime', self.num_samples)
+
+        if visualization:
+            plt.show()
+
+
+def auto_choose_num_features_to_show(lime_interpreter, label, percentage_to_show):
+    segments = lime_interpreter.segments
+    lime_weights = lime_interpreter.local_weights[label]
+    num_pixels_threshold_in_a_sp = segments.shape[0] * segments.shape[1] // len(np.unique(segments)) // 8
+
+    # l1 norm with filtered weights.
+    used_weights = [(tuple_w[0], tuple_w[1]) for i, tuple_w in enumerate(lime_weights) if tuple_w[1] > 0]
+    norm = np.sum([tuple_w[1] for i, tuple_w in enumerate(used_weights)])
+    normalized_weights = [(tuple_w[0], tuple_w[1] / norm) for i, tuple_w in enumerate(lime_weights)]
+
+    a = 0.0
+    n = 0
+    for i, tuple_w in enumerate(normalized_weights):
+        if tuple_w[1] < 0:
+            continue
+        if len(np.where(segments == tuple_w[0])[0]) < num_pixels_threshold_in_a_sp:
+            continue
+
+        a += tuple_w[1]
+        if a > percentage_to_show:
+            n = i + 1
+            break
+
+    if percentage_to_show <= 0.0:
+        return 5
+
+    if n == 0:
+        return auto_choose_num_features_to_show(lime_interpreter, label, percentage_to_show-0.1)
+
+    return n
+
+
+def get_cam(image_show, feature_maps, fc_weights, label_index, cam_min=None, cam_max=None):
+    _, nc, h, w = feature_maps.shape
+
+    cam = feature_maps * fc_weights[:, label_index].reshape(1, nc, 1, 1)
+    cam = cam.sum((0, 1))
+
+    if cam_min is None:
+        cam_min = np.min(cam)
+    if cam_max is None:
+        cam_max = np.max(cam)
+
+    cam = cam - cam_min
+    cam = cam / cam_max
+    cam = np.uint8(255 * cam)
+    cam_img = cv2.resize(cam, image_show.shape[0:2], interpolation=cv2.INTER_LINEAR)
+
+    heatmap = cv2.applyColorMap(np.uint8(255 * cam_img), cv2.COLORMAP_JET)
+    heatmap = np.float32(heatmap)
+    cam = heatmap + np.float32(image_show)
+    cam = cam / np.max(cam)
+
+    return cam
+
+
+def save_fig(data_, save_outdir, algorithm_name, num_samples=3000):
+    import matplotlib.pyplot as plt
+    if isinstance(data_, str):
+        if algorithm_name == 'cam':
+            f_out = f"{algorithm_name}_{data_.split('/')[-1]}.png"
+        else:
+            f_out = f"{algorithm_name}_{data_.split('/')[-1]}_s{num_samples}.png"
+        plt.savefig(
+            os.path.join(save_outdir, f_out)
+        )
+    else:
+        n = 0
+        if algorithm_name == 'cam':
+            f_out = f'cam-{n}.png'
+        else:
+            f_out = f'{algorithm_name}_s{num_samples}-{n}.png'
+        while os.path.exists(
+                os.path.join(save_outdir, f_out)
+        ):
+            n += 1
+            if algorithm_name == 'cam':
+                f_out = f'cam-{n}.png'
+            else:
+                f_out = f'{algorithm_name}_s{num_samples}-{n}.png'
+            continue
+        plt.savefig(
+            os.path.join(
+                save_outdir, f_out
+            )
+        )
+    print('The image of intrepretation result save in {}'.format(os.path.join(
+                save_outdir, f_out
+            )))

paddlex/interpret/core/normlime_base.py

+#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+import os
+import numpy as np
+import glob
+
+from ..as_data_reader.readers import read_image
+from . import lime_base
+from ._session_preparation import compute_features_for_kmeans, h_pre_models_kmeans
+
+
+def load_kmeans_model(fname):
+    import pickle
+    with open(fname, 'rb') as f:
+        kmeans_model = pickle.load(f)
+
+    return kmeans_model
+
+
+def combine_normlime_and_lime(lime_weights, g_weights):
+    pred_labels = lime_weights.keys()
+    combined_weights = {y: [] for y in pred_labels}
+
+    for y in pred_labels:
+        normlized_lime_weights_y = lime_weights[y]
+        lime_weights_dict = {tuple_w[0]: tuple_w[1] for tuple_w in normlized_lime_weights_y}
+
+        normlized_g_weight_y = g_weights[y]
+        normlime_weights_dict = {tuple_w[0]: tuple_w[1] for tuple_w in normlized_g_weight_y}
+
+        combined_weights[y] = [
+            (seg_k, lime_weights_dict[seg_k] * normlime_weights_dict[seg_k])
+            for seg_k in lime_weights_dict.keys()
+        ]
+
+        combined_weights[y] = sorted(combined_weights[y],
+                                     key=lambda x: np.abs(x[1]), reverse=True)
+
+    return combined_weights
+
+
+def avg_using_superpixels(features, segments):
+    one_list = np.zeros((len(np.unique(segments)), features.shape[2]))
+    for x in np.unique(segments):
+        one_list[x] = np.mean(features[segments == x], axis=0)
+
+    return one_list
+
+
+def centroid_using_superpixels(features, segments):
+    from skimage.measure import regionprops
+    regions = regionprops(segments + 1)
+    one_list = np.zeros((len(np.unique(segments)), features.shape[2]))
+    for i, r in enumerate(regions):
+        one_list[i] = features[int(r.centroid[0] + 0.5), int(r.centroid[1] + 0.5), :]
+    # print(one_list.shape)
+    return one_list
+
+
+def get_feature_for_kmeans(feature_map, segments):
+    from sklearn.preprocessing import normalize
+    centroid_feature = centroid_using_superpixels(feature_map, segments)
+    avg_feature = avg_using_superpixels(feature_map, segments)
+    x = np.concatenate((centroid_feature, avg_feature), axis=-1)
+    x = normalize(x)
+    return x
+
+
+def precompute_normlime_weights(list_data_, predict_fn, num_samples=3000, batch_size=50, save_dir='./tmp'):
+    # save lime weights and kmeans cluster labels
+    precompute_lime_weights(list_data_, predict_fn, num_samples, batch_size, save_dir)
+
+    # load precomputed results, compute normlime weights and save.
+    fname_list = glob.glob(os.path.join(save_dir, f'lime_weights_s{num_samples}*.npy'))
+    return compute_normlime_weights(fname_list, save_dir, num_samples)
+
+
+def save_one_lime_predict_and_kmean_labels(lime_all_weights, image_pred_labels, cluster_labels, save_path):
+
+    lime_weights = {}
+    for label in image_pred_labels:
+        lime_weights[label] = lime_all_weights[label]
+
+    for_normlime_weights = {
+        'lime_weights': lime_weights,  # a dict: class_label: (seg_label, weight)
+        'cluster': cluster_labels  # a list with segments as indices.
+    }
+
+    np.save(save_path, for_normlime_weights)
+
+
+def precompute_lime_weights(list_data_, predict_fn, num_samples, batch_size, save_dir):
+    kmeans_model = load_kmeans_model(h_pre_models_kmeans)
+
+    for data_index, each_data_ in enumerate(list_data_):
+        if isinstance(each_data_, str):
+            save_path = f"lime_weights_s{num_samples}_{each_data_.split('/')[-1].split('.')[0]}.npy"
+            save_path = os.path.join(save_dir, save_path)
+        else:
+            save_path = f"lime_weights_s{num_samples}_{data_index}.npy"
+            save_path = os.path.join(save_dir, save_path)
+
+        if os.path.exists(save_path):
+            print(f'{save_path} exists, not computing this one.')
+            continue
+
+        print('processing', each_data_ if isinstance(each_data_, str) else data_index,
+              f', {data_index}/{len(list_data_)}')
+
+        image_show = read_image(each_data_)
+        result = predict_fn(image_show)
+        result = result[0]  # only one image here.
+
+        if abs(np.sum(result) - 1.0) > 1e-4:
+            # softmax
+            exp_result = np.exp(result)
+            probability = exp_result / np.sum(exp_result)
+        else:
+            probability = result
+
+        pred_label = np.argsort(probability)[::-1]
+
+        # top_k = argmin(top_n) > threshold
+        threshold = 0.05
+        top_k = 0
+        for l in pred_label:
+            if probability[l] < threshold or top_k == 5:
+                break
+            top_k += 1
+
+        if top_k == 0:
+            top_k = 1
+
+        pred_label = pred_label[:top_k]
+
+        algo = lime_base.LimeImageInterpreter()
+        interpreter = algo.interpret_instance(image_show[0], predict_fn, pred_label, 0,
+                                          num_samples=num_samples, batch_size=batch_size)
+
+        cluster_labels = kmeans_model.predict(
+            get_feature_for_kmeans(compute_features_for_kmeans(image_show).transpose((1, 2, 0)), interpreter.segments)
+        )
+        save_one_lime_predict_and_kmean_labels(
+            interpreter.local_weights, pred_label,
+            cluster_labels,
+            save_path
+        )
+
+
+def compute_normlime_weights(a_list_lime_fnames, save_dir, lime_num_samples):
+    normlime_weights_all_labels = {}
+    for f in a_list_lime_fnames:
+        try:
+            lime_weights_and_cluster = np.load(f, allow_pickle=True).item()
+            lime_weights = lime_weights_and_cluster['lime_weights']
+            cluster = lime_weights_and_cluster['cluster']
+        except:
+            print('When loading precomputed LIME result, skipping', f)
+            continue
+        print('Loading precomputed LIME result,', f)
+
+        pred_labels = lime_weights.keys()
+        for y in pred_labels:
+            normlime_weights = normlime_weights_all_labels.get(y, {})
+            w_f_y = [abs(w[1]) for w in lime_weights[y]]
+            w_f_y_l1norm = sum(w_f_y)
+
+            for w in lime_weights[y]:
+                seg_label = w[0]
+                weight = w[1] * w[1] / w_f_y_l1norm
+                a = normlime_weights.get(cluster[seg_label], [])
+                a.append(weight)
+                normlime_weights[cluster[seg_label]] = a
+
+            normlime_weights_all_labels[y] = normlime_weights
+
+    # compute normlime
+    for y in normlime_weights_all_labels:
+        normlime_weights = normlime_weights_all_labels.get(y, {})
+        for k in normlime_weights:
+            normlime_weights[k] = sum(normlime_weights[k]) / len(normlime_weights[k])
+
+    # check normlime
+    if len(normlime_weights_all_labels.keys()) < max(normlime_weights_all_labels.keys()) + 1:
+        print(
+            "\n"
+            "Warning: !!! \n"
+            f"There are at least {max(normlime_weights_all_labels.keys()) + 1} classes, "
+            f"but the NormLIME has results of only {len(normlime_weights_all_labels.keys())} classes. \n"
+            "It may have cause unstable results in the later computation"
+            " but can be improved by computing more test samples."
+            "\n"
+        )
+
+    n = 0
+    f_out = f'normlime_weights_s{lime_num_samples}_samples_{len(a_list_lime_fnames)}-{n}.npy'
+    while os.path.exists(
+            os.path.join(save_dir, f_out)
+    ):
+        n += 1
+        f_out = f'normlime_weights_s{lime_num_samples}_samples_{len(a_list_lime_fnames)}-{n}.npy'
+        continue
+
+    np.save(
+        os.path.join(save_dir, f_out),
+        normlime_weights_all_labels
+    )
+    return os.path.join(save_dir, f_out)
+

paddlex/interpret/interpretation_predict.py

+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+def interpretation_predict(model, images):
+    model.arrange_transforms(
+            transforms=model.test_transforms, mode='test')
+    new_imgs = []
+    for i in range(images.shape[0]):
+        img = images[i]
+        new_imgs.append(model.test_transforms(img)[0])
+    new_imgs = np.array(new_imgs)
+    result = model.exe.run(
+        model.test_prog,
+        feed={'image': new_imgs},
+        fetch_list=list(model.interpretation_feats.values()))
+    return result
\ No newline at end of file

paddlex/interpret/visualize.py

+#copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+import os
+import cv2
+import copy
+import os.path as osp
+import numpy as np
+import paddlex as pdx
+from .interpretation_predict import interpretation_predict
+from .core.interpretation import Interpretation
+from .core.normlime_base import precompute_normlime_weights
+
+
+def visualize(img_file, 
+              model, 
+              dataset=None,
+              algo='lime',
+              num_samples=3000, 
+              batch_size=50,
+              save_dir='./'):
+    """可解释性可视化。
+    Args:
+        img_file (str): 预测图像路径。
+        model (paddlex.cv.models): paddlex中的模型。
+        dataset (paddlex.datasets): 数据集读取器，默认为None。
+        algo (str): 可解释性方式，当前可选'lime'和'normlime'。
+        num_samples (int): LIME用于学习线性模型的采样数，默认为3000。
+        batch_size (int): 预测数据batch大小，默认为50。
+        save_dir (str): 可解释性可视化结果（保存为png格式文件）和中间文件存储路径。        
+    """
+    assert model.model_type == 'classifier', \
+        'Now the interpretation visualize only be supported in classifier!'
+    if model.status != 'Normal':
+        raise Exception('The interpretation only can deal with the Normal model')
+    if not osp.exists(save_dir):
+        os.makedirs(save_dir)
+    model.arrange_transforms(
+                transforms=model.test_transforms, mode='test')
+    tmp_transforms = copy.deepcopy(model.test_transforms)
+    tmp_transforms.transforms = tmp_transforms.transforms[:-2]
+    img = tmp_transforms(img_file)[0]
+    img = np.around(img).astype('uint8')
+    img = np.expand_dims(img, axis=0)
+    interpreter = None
+    if algo == 'lime':
+        interpreter = get_lime_interpreter(img, model, dataset, num_samples=num_samples, batch_size=batch_size)
+    elif algo == 'normlime':
+        if dataset is None:
+            raise Exception('The dataset is None. Cannot implement this kind of interpretation')
+        interpreter = get_normlime_interpreter(img, model, dataset, 
+                                     num_samples=num_samples, batch_size=batch_size,
+                                     save_dir=save_dir)
+    else:
+        raise Exception('The {} interpretation method is not supported yet!'.format(algo))
+    img_name = osp.splitext(osp.split(img_file)[-1])[0]
+    interpreter.interpret(img, save_dir=save_dir)
+    
+    
+def get_lime_interpreter(img, model, dataset, num_samples=3000, batch_size=50):
+    def predict_func(image):
+        image = image.astype('float32')
+        for i in range(image.shape[0]):
+            image[i] = cv2.cvtColor(image[i], cv2.COLOR_RGB2BGR)
+        tmp_transforms = copy.deepcopy(model.test_transforms.transforms)
+        model.test_transforms.transforms = model.test_transforms.transforms[-2:]
+        out = interpretation_predict(model, image)
+        model.test_transforms.transforms = tmp_transforms
+        return out[0]
+    labels_name = None
+    if dataset is not None:
+        labels_name = dataset.labels
+    interpreter = Interpretation('lime', 
+                            predict_func,
+                            labels_name,
+                            num_samples=num_samples, 
+                            batch_size=batch_size)
+    return interpreter
+
+
+def get_normlime_interpreter(img, model, dataset, num_samples=3000, batch_size=50, save_dir='./'):
+    def precompute_predict_func(image):
+        image = image.astype('float32')
+        tmp_transforms = copy.deepcopy(model.test_transforms.transforms)
+        model.test_transforms.transforms = model.test_transforms.transforms[-2:]
+        out = interpretation_predict(model, image)
+        model.test_transforms.transforms = tmp_transforms
+        return out[0]
+    def predict_func(image):
+        image = image.astype('float32')
+        for i in range(image.shape[0]):
+            image[i] = cv2.cvtColor(image[i], cv2.COLOR_RGB2BGR)
+        tmp_transforms = copy.deepcopy(model.test_transforms.transforms)
+        model.test_transforms.transforms = model.test_transforms.transforms[-2:]
+        out = interpretation_predict(model, image)
+        model.test_transforms.transforms = tmp_transforms
+        return out[0]
+    labels_name = None
+    if dataset is not None:
+        labels_name = dataset.labels
+    root_path = os.environ['HOME']
+    root_path = osp.join(root_path, '.paddlex')
+    pre_models_path = osp.join(root_path, "pre_models")
+    if not osp.exists(pre_models_path):
+        if not osp.exists(root_path):
+            os.makedirs(root_path)
+        url = "https://bj.bcebos.com/paddlex/interpret/pre_models.tar.gz"
+        pdx.utils.download_and_decompress(url, path=root_path)
+    npy_dir = precompute_for_normlime(precompute_predict_func, 
+                                      dataset, 
+                                      num_samples=num_samples, 
+                                      batch_size=batch_size,
+                                      save_dir=save_dir)
+    interpreter = Interpretation('normlime', 
+                            predict_func,
+                            labels_name,
+                            num_samples=num_samples, 
+                            batch_size=batch_size,
+                            normlime_weights=npy_dir)
+    return interpreter
+
+
+def precompute_for_normlime(predict_func, dataset, num_samples=3000, batch_size=50, save_dir='./'):
+    image_list = []
+    for item in dataset.file_list:
+        image_list.append(item[0])
+    return precompute_normlime_weights(
+            image_list,  
+            predict_func,
+            num_samples=num_samples, 
+            batch_size=batch_size,
+            save_dir=save_dir)
+  

paddlex/tools/__init__.py

@@ -14,11 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .x2imagenet import EasyData2ImageNet
-from .x2coco import LabelMe2COCO
-from .x2coco import EasyData2COCO
-from .x2voc import LabelMe2VOC
-from .x2voc import EasyData2VOC
-from .x2seg import JingLing2Seg
-from .x2seg import LabelMe2Seg
-from .x2seg import EasyData2Seg
+from .convert import *
\ No newline at end of file

paddlex/tools/convert.py

+#!/usr/bin/env python
+# coding: utf-8
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .x2imagenet import EasyData2ImageNet
+from .x2coco import LabelMe2COCO
+from .x2coco import EasyData2COCO
+from .x2voc import LabelMe2VOC
+from .x2voc import EasyData2VOC
+from .x2seg import JingLing2Seg
+from .x2seg import LabelMe2Seg
+from .x2seg import EasyData2Seg
+
+easydata2imagenet = EasyData2ImageNet().convert
+labelme2coco = LabelMe2COCO().convert
+easydata2coco = EasyData2COCO().convert
+labelme2voc = LabelMe2VOC().convert
+easydata2voc = EasyData2VOC().convert
+jingling2seg = JingLing2Seg().convert
+labelme2seg = LabelMe2Seg().convert
+easydata2seg = EasyData2Seg().convert

setup.py

@@ -30,7 +30,7 @@ setuptools.setup(
     setup_requires=['cython', 'numpy'],
     install_requires=[
         "pycocotools;platform_system!='Windows'", 'pyyaml', 'colorama', 'tqdm',
-        'paddleslim==1.0.1', 'visualdl==2.0.0a2'
+        'paddleslim==1.0.1', 'visualdl>=2.0.0a2'
     ],
     classifiers=[
         "Programming Language :: Python :: 3",

tutorials/interpret/interpret.py

+import os
+# 选择使用0号卡
+os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+import os.path as osp
+import paddlex as pdx
+
+# 下载和解压Imagenet果蔬分类数据集
+veg_dataset = 'https://bj.bcebos.com/paddlex/interpret/mini_imagenet_veg.tar.gz'
+pdx.utils.download_and_decompress(veg_dataset, path='./')
+
+# 下载和解压已训练好的MobileNetV2模型
+model_file = 'https://bj.bcebos.com/paddlex/interpret/mini_imagenet_veg_mobilenetv2.tar.gz'
+pdx.utils.download_and_decompress(model_file, path='./')
+
+# 加载模型
+model = pdx.load_model('mini_imagenet_veg_mobilenetv2')
+
+# 定义测试所用的数据集
+test_dataset = pdx.datasets.ImageNet(
+    data_dir='mini_imagenet_veg',
+    file_list=osp.join('mini_imagenet_veg', 'test_list.txt'),
+    label_list=osp.join('mini_imagenet_veg', 'labels.txt'),
+    transforms=model.test_transforms)
+
+# 可解释性可视化
+pdx.interpret.visualize(
+         'mini_imagenet_veg/mushroom/n07734744_1106.JPEG', 
+          model,
+          test_dataset, 
+          algo='lime',
+          save_dir='./')
+pdx.interpret.visualize(
+         'mini_imagenet_veg/mushroom/n07734744_1106.JPEG', 
+          model, 
+          test_dataset, 
+          algo='normlime',
+          save_dir='./')