Merge pull request #2219 from cuicheng01/fix_multilabel

fix multilabel

deploy/configs/inference_cls_multilabel.yaml

@@ -25,11 +25,12 @@ PreProcess:
         order: ''
         channel_num: 3
     - ToCHWImage:
-    
+
 PostProcess:
-  main_indicator: MultiLabelTopk
-  MultiLabelTopk:
-    topk: 5
-    class_id_map_file: None
+  main_indicator: MultiLabelThreshOutput
+  MultiLabelThreshOutput:
+    threshold: 0.5
+    class_id_map_file: "../ppcls/utils/NUS-WIDE-SCENE_label_list.txt"
+    delimiter: " "
   SavePreLabel:
-    save_dir: ./pre_label/
+    save_dir: ./pre_label/
\ No newline at end of file

deploy/python/postprocess.py

@@ -82,10 +82,11 @@ class ThreshOutput(object):
 
 
 class Topk(object):
-    def __init__(self, topk=1, class_id_map_file=None):
+    def __init__(self, topk=1, class_id_map_file=None, delimiter=None):
         assert isinstance(topk, (int, ))
-        self.class_id_map = self.parse_class_id_map(class_id_map_file)
         self.topk = topk
+        self.class_id_map = self.parse_class_id_map(class_id_map_file)
+        self.delimiter = delimiter if delimiter is not None else " "
 
     def parse_class_id_map(self, class_id_map_file):
         if class_id_map_file is None:
@@ -102,21 +103,20 @@ class Topk(object):
             with open(class_id_map_file, "r") as fin:
                 lines = fin.readlines()
                 for line in lines:
-                    partition = line.split("\n")[0].partition(" ")
+                    partition = line.split("\n")[0].partition(self.delimiter)
                     class_id_map[int(partition[0])] = str(partition[-1])
         except Exception as ex:
             print(ex)
             class_id_map = None
         return class_id_map
 
-    def __call__(self, x, file_names=None, multilabel=False):
+    def __call__(self, x, file_names=None):
         if file_names is not None:
             assert x.shape[0] == len(file_names)
         y = []
         for idx, probs in enumerate(x):
             index = probs.argsort(axis=0)[-self.topk:][::-1].astype(
-                "int32") if not multilabel else np.where(
-                    probs >= 0.5)[0].astype("int32")
+                "int32")
             clas_id_list = []
             score_list = []
             label_name_list = []
@@ -138,12 +138,56 @@ class Topk(object):
         return y
 
 
-class MultiLabelTopk(Topk):
-    def __init__(self, topk=1, class_id_map_file=None):
-        super().__init__()
+class MultiLabelThreshOutput(object):
+    def __init__(self, threshold=0.5, class_id_map_file=None, delimiter=None):
+        self.threshold = threshold
+        self.delimiter = delimiter if delimiter is not None else " "
+        self.class_id_map = self.parse_class_id_map(class_id_map_file)
+
+    def parse_class_id_map(self, class_id_map_file):
+        if class_id_map_file is None:
+            return None
+
+        if not os.path.exists(class_id_map_file):
+            print(
+                "Warning: If want to use your own label_dict, please input legal path!\nOtherwise label_names will be empty!"
+            )
+            return None
+
+        try:
+            class_id_map = {}
+            with open(class_id_map_file, "r") as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    partition = line.split("\n")[0].partition(self.delimiter)
+                    class_id_map[int(partition[0])] = str(partition[-1])
+        except Exception as ex:
+            print(ex)
+            class_id_map = None
+        return class_id_map
 
     def __call__(self, x, file_names=None):
-        return super().__call__(x, file_names, multilabel=True)
+        y = []
+        for idx, probs in enumerate(x):
+            index = np.where(probs >= self.threshold)[0].astype("int32")
+            clas_id_list = []
+            score_list = []
+            label_name_list = []
+            for i in index:
+                clas_id_list.append(i.item())
+                score_list.append(probs[i].item())
+                if self.class_id_map is not None:
+                    label_name_list.append(self.class_id_map[i.item()])
+            result = {
+                "class_ids": clas_id_list,
+                "scores": np.around(
+                    score_list, decimals=5).tolist(),
+                "label_names": label_name_list
+            }
+            if file_names is not None:
+                result["file_name"] = file_names[idx]
+            y.append(result)
+        return y
 
 
 class SavePreLabel(object):

docs/en/quick_start/quick_start_multilabel_classification_en.md

@@ -51,6 +51,9 @@ python3 -m paddle.distributed.launch \
 
 After training 10 epochs, the best correctness of the validation set should be around 0.95.
 
+**Note:**
+1. Currently, the loss function for multi-label classification only supports `MultiLabelLoss` (BCE Loss).
+2. Currently, the evaluation metrics of multi-label classification support `Accuracy_Score` and `HammingDistance`. Please look forward to the support of other evaluation metrics.
 
 <a name="3"></a>
 ## 3. Model Evaluation
@@ -74,7 +77,7 @@ python3 tools/infer.py \
 Obtain an output silimar to the following:
 
 ```
-[{'class_ids': [6, 13, 17, 23, 26, 30], 'scores': [0.95683, 0.5567, 0.55211, 0.99088, 0.5943, 0.78767], 'file_name': './deploy/images/0517_2715693311.jpg', 'label_names': []}]
+[{'class_ids': [6, 13, 17, 23, 30], 'scores': [0.98217, 0.78129, 0.64377, 0.9942, 0.96109], 'label_names': ['clouds', 'lake', 'ocean', 'sky', 'water'], 'file_name': 'deploy/images/0517_2715693311.jpg'}]
 ```
 
 
@@ -107,11 +110,14 @@ Inference and prediction through predictive engines:
 
 ```
 python3 python/predict_cls.py \
-     -c configs/inference_multilabel_cls.yaml
+     -c configs/inference_cls_multilabel.yaml
 ```
+The predicted pictures are as follows:
 
-Obtain an output silimar to the following:
+![](../../images/quick_start/multi_label_demo.png)
+
+After executing the inference command, obtain an output silimar to the following:
 
 ```
-0517_2715693311.jpg:    class id(s): [6, 13, 17, 23, 26, 30], score(s): [0.96, 0.56, 0.55, 0.99, 0.59, 0.79], label_name(s): []
+0517_2715693311.jpg:    class id(s): [6, 13, 17, 23, 30], score(s): [0.98, 0.78, 0.64, 0.99, 0.96], label_name(s): ['clouds', 'lake', 'ocean', 'sky', 'water']
 ```

docs/zh_CN/quick_start/quick_start_multilabel_classification.md

@@ -50,6 +50,10 @@ python3 -m paddle.distributed.launch \
 
 训练 10 epoch 之后，验证集最好的正确率应该在 0.95 左右。
 
+**注意:** 
+1. 目前多标签分类的损失函数仅支持`MultiLabelLoss`(BCE Loss)。
+2. 目前多标签分类的评估指标支持`AccuracyScore`和`HammingDistance`,其他评估指标敬请期待。
+
 <a name="3"></a>
 
 ## 3. 模型评估
@@ -70,8 +74,8 @@ python3 tools/infer.py \
 ```
 
 得到类似下面的输出：
-```  
-[{'class_ids': [6, 13, 17, 23, 26, 30], 'scores': [0.95683, 0.5567, 0.55211, 0.99088, 0.5943, 0.78767], 'file_name': './deploy/images/0517_2715693311.jpg', 'label_names': []}]
+```
+[{'class_ids': [6, 13, 17, 23, 30], 'scores': [0.98217, 0.78129, 0.64377, 0.9942, 0.96109], 'label_names': ['clouds', 'lake', 'ocean', 'sky', 'water'], 'file_name': 'deploy/images/0517_2715693311.jpg'}]
 ```
 
 <a name="5"></a>
@@ -100,10 +104,13 @@ cd ./deploy
 
 ```
 python3 python/predict_cls.py \
-     -c configs/inference_multilabel_cls.yaml
+     -c configs/inference_cls_multilabel.yaml
 ```
+推理图片如下：
 
-得到类似下面的输出：
+![](../../images/quick_start/multi_label_demo.png)
+
+执行推理命令后，得到类似下面的输出：
 ```
-0517_2715693311.jpg:    class id(s): [6, 13, 17, 23, 26, 30], score(s): [0.96, 0.56, 0.55, 0.99, 0.59, 0.79], label_name(s): []
+0517_2715693311.jpg:    class id(s): [6, 13, 17, 23, 30], score(s): [0.98, 0.78, 0.64, 0.99, 0.96], label_name(s): ['clouds', 'lake', 'ocean', 'sky', 'water']
 ```

ppcls/configs/quick_start/professional/MobileNetV1_multilabel.yaml

@@ -99,7 +99,7 @@ DataLoader:
       use_shared_memory: True
 
 Infer:
-  infer_imgs: ./deploy/images/0517_2715693311.jpg
+  infer_imgs: deploy/images/0517_2715693311.jpg
   batch_size: 10
   transforms:
     - DecodeImage:
@@ -116,9 +116,10 @@ Infer:
         order: ''
     - ToCHWImage:
   PostProcess:
-    name: MultiLabelTopk
-    topk: 5
-    class_id_map_file: None
+    name: MultiLabelThreshOutput
+    threshold: 0.5
+    class_id_map_file: "ppcls/utils/NUS-WIDE-SCENE_label_list.txt"
+    delimiter: " "
 
 Metric:
   Train:

ppcls/data/postprocess/__init__.py

@@ -16,9 +16,10 @@ import importlib
 
 from . import topk, threshoutput
 
-from .topk import Topk, MultiLabelTopk
-from .threshoutput import ThreshOutput
-from .attr_rec import VehicleAttribute, PersonAttribute, TableAttribute
+from .topk import Topk
+from .threshoutput import ThreshOutput, MultiLabelThreshOutput
+from .attr_rec import VehicleAttribute, PersonAttribute
+
 
 
 def build_postprocess(config):

ppcls/data/postprocess/threshoutput.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import numpy as np
 import paddle.nn.functional as F
 
 
@@ -34,3 +36,55 @@ class ThreshOutput(object):
                 result["file_name"] = file_names[idx]
             y.append(result)
         return y
+
+
+class MultiLabelThreshOutput(object):
+    def __init__(self, threshold=0.5, class_id_map_file=None, delimiter=None):
+        self.threshold = threshold
+        self.delimiter = delimiter if delimiter is not None else " "
+        self.class_id_map = self.parse_class_id_map(class_id_map_file)
+
+    def parse_class_id_map(self, class_id_map_file):
+        if class_id_map_file is None:
+            return None
+        if not os.path.exists(class_id_map_file):
+            print(
+                "Warning: If want to use your own label_dict, please input legal path!\nOtherwise label_names will be empty!"
+            )
+            return None
+
+        try:
+            class_id_map = {}
+            with open(class_id_map_file, "r") as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    partition = line.split("\n")[0].partition(self.delimiter)
+                    class_id_map[int(partition[0])] = str(partition[-1])
+        except Exception as ex:
+            print(ex)
+            class_id_map = None
+        return class_id_map
+
+    def __call__(self, x, file_names=None):
+        y = []
+        x = F.sigmoid(x).numpy()
+        for idx, probs in enumerate(x):
+            index = np.where(probs >= self.threshold)[0].astype("int32")
+            clas_id_list = []
+            score_list = []
+            label_name_list = []
+            for i in index:
+                clas_id_list.append(i.item())
+                score_list.append(probs[i].item())
+                if self.class_id_map is not None:
+                    label_name_list.append(self.class_id_map[i.item()])
+            result = {
+                "class_ids": clas_id_list,
+                "scores": np.around(
+                    score_list, decimals=5).tolist(),
+                "label_names": label_name_list    
+            }
+            if file_names is not None:
+                result["file_name"] = file_names[idx]
+            y.append(result)
+        return y

ppcls/data/postprocess/topk.py

@@ -46,19 +46,18 @@ class Topk(object):
             class_id_map = None
         return class_id_map
 
-    def __call__(self, x, file_names=None, multilabel=False):
+    def __call__(self, x, file_names=None):
         if isinstance(x, dict):
             x = x['logits']
         assert isinstance(x, paddle.Tensor)
         if file_names is not None:
             assert x.shape[0] == len(file_names)
-        x = F.softmax(x, axis=-1) if not multilabel else F.sigmoid(x)
+        x = F.softmax(x, axis=-1)
         x = x.numpy()
         y = []
         for idx, probs in enumerate(x):
             index = probs.argsort(axis=0)[-self.topk:][::-1].astype(
-                "int32") if not multilabel else np.where(
-                    probs >= 0.5)[0].astype("int32")
+                "int32")
             clas_id_list = []
             score_list = []
             label_name_list = []
@@ -79,10 +78,3 @@ class Topk(object):
             y.append(result)
         return y
 
-
-class MultiLabelTopk(Topk):
-    def __init__(self, topk=1, class_id_map_file=None):
-        super().__init__()
-
-    def __call__(self, x, file_names=None):
-        return super().__call__(x, file_names, multilabel=True)

ppcls/engine/engine.py

@@ -502,7 +502,7 @@ class Engine(object):
         assert self.mode == "export"
         use_multilabel = self.config["Global"].get(
             "use_multilabel",
-            False) and "ATTRMetric" in self.config["Metric"]["Eval"][0]
+            False) or "ATTRMetric" in self.config["Metric"]["Eval"][0]
         model = ExportModel(self.config["Arch"], self.model, use_multilabel)
         if self.config["Global"]["pretrained_model"] is not None:
             load_dygraph_pretrain(model.base_model,

ppcls/utils/NUS-WIDE-SCENE_label_list.txt

+0 airport
+1 beach
+2 bridge
+3 buildings
+4 castle
+5 cityscape
+6 clouds
+7 frost
+8 garden
+9 glacier
+10 grass
+11 harbor
+12 house
+13 lake
+14 moon
+15 mountain
+16 nighttime
+17 ocean
+18 plants
+19 railroad
+20 rainbow
+21 reflection
+22 road
+23 sky
+24 snow
+25 street
+26 sunset
+27 temple
+28 town
+29 valley
+30 water
+31 waterfall
+32 window