Merge branch 'develop' of https://github.com/PaddlePaddle/PaddleHub into develop

demo/ernie-classification/finetune_with_hub.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Finetuning on classification tasks."""
+"""Finetuning on classification task """
 
 from __future__ import absolute_import
 from __future__ import division
@@ -40,20 +40,24 @@ args = parser.parse_args()
 # yapf: enable.
 
 if __name__ == '__main__':
+    # Select a finetune strategy
     strategy = hub.BERTFinetuneStrategy(
         weight_decay=args.weight_decay,
         learning_rate=args.learning_rate,
         warmup_strategy="linear_warmup_decay",
     )
+
+    # Setup runing config for PaddleHub Finetune API
     config = hub.RunConfig(
         use_cuda=True,
         num_epoch=args.num_epoch,
         batch_size=args.batch_size,
         strategy=strategy)
 
-    # loading Paddlehub BERT
+    # loading Paddlehub ERNIE pretrained model
     module = hub.Module(name="ernie")
 
+    # Sentence classification  dataset reader
     reader = hub.reader.ClassifyReader(
         dataset=hub.dataset.ChnSentiCorp(),  # download chnsenticorp dataset
         vocab_path=module.get_vocab_path(),
@@ -72,7 +76,7 @@ if __name__ == '__main__':
         pooled_output = output_dict["pooled_output"]
 
         # Setup feed list for data feeder
-        # Must feed all the tensor of bert's module need
+        # Must feed all the tensor of ERNIE's module need
         feed_list = [
             input_dict["input_ids"].name, input_dict["position_ids"].name,
             input_dict["segment_ids"].name, input_dict["input_mask"].name,

demo/ernie-seq-labeling/finetune_with_hub.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Finetuning on classification tasks."""
+"""Finetuning on sequence labeling task."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -40,11 +40,14 @@ args = parser.parse_args()
 # yapf: enable.
 
 if __name__ == '__main__':
+    # Select a finetune strategy
     strategy = hub.BERTFinetuneStrategy(
         weight_decay=args.weight_decay,
         learning_rate=args.learning_rate,
         warmup_strategy="linear_warmup_decay",
     )
+
+    # Setup runing config for PaddleHub Finetune API
     config = hub.RunConfig(
         eval_interval=100,
         use_cuda=True,
@@ -52,9 +55,10 @@ if __name__ == '__main__':
         batch_size=args.batch_size,
         strategy=strategy)
 
-    # loading Paddlehub ERNIE
+    # loading Paddlehub ERNIE pretrained model
     module = hub.Module(name="ernie")
 
+    # Sequence Label dataset reader
     reader = hub.reader.SequenceLabelReader(
         dataset=hub.dataset.MSRA_NER(),
         vocab_path=module.get_vocab_path(),
@@ -75,13 +79,13 @@ if __name__ == '__main__':
         sequence_output = output_dict["sequence_output"]
 
         # Setup feed list for data feeder
-        # Must feed all the tensor of bert's module need
+        # Must feed all the tensor of ERNIE's module need
         feed_list = [
             input_dict["input_ids"].name, input_dict["position_ids"].name,
             input_dict["segment_ids"].name, input_dict["input_mask"].name,
             label.name, seq_len
         ]
-        # Define a classfication finetune task by PaddleHub's API
+        # Define a sequence labeling finetune task by PaddleHub's API
         seq_label_task = hub.create_seq_labeling_task(
             feature=sequence_output,
             labels=label,

paddlehub/finetune/finetune.py

@@ -27,6 +27,8 @@ import numpy as np
 from paddlehub.common.logger import logger
 from paddlehub.finetune.strategy import BERTFinetuneStrategy, DefaultStrategy
 from paddlehub.finetune.checkpoint import load_checkpoint, save_checkpoint
+from paddlehub.finetune.evaluate import evaluate_cls_task,
+evaluate_seq_labeling_task
 from visualdl import LogWriter
 import paddlehub as hub
 
@@ -135,17 +137,17 @@ def _finetune_seq_label_task(task,
                         exe=exe)
 
                 if do_eval and global_step % config.eval_interval == 0:
-                    evaluate_seq_label(
+                    evaluate_seq_label_task(
                         task,
                         data_reader,
                         feed_list,
-                        phase="dev",
+                        phase="test",
                         config=config)
-                    evaluate_seq_label(
+                    evaluate_seq_label_task(
                         task,
                         data_reader,
                         feed_list,
-                        phase="test",
+                        phase="dev",
                         config=config)
 
         # NOTE: current saved checkpoint machanism is not completed, it can't
@@ -157,52 +159,13 @@ def _finetune_seq_label_task(task,
             exe=exe)
 
         if do_eval:
-            evaluate_seq_label(
+            evaluate_seq_label_task(
+                task, data_reader, feed_list, phase="dev", config=config)
+            evaluate_seq_label_task(
                 task, data_reader, feed_list, phase="test", config=config)
         logger.info("PaddleHub finetune finished.")
 
 
-def evaluate_seq_label(task, data_reader, feed_list, phase="test", config=None):
-    fetch_list = [
-        task.variable("labels").name,
-        task.variable("infers").name,
-        task.variable("seq_len").name,
-        task.variable("loss").name
-    ]
-    logger.info("Evaluation on {} dataset start".format(phase))
-    inference_program = task.inference_program()
-    batch_size = config.batch_size
-    place, dev_count = _get_running_device_info(config)
-    exe = fluid.Executor(place=place)
-    with fluid.program_guard(inference_program):
-        data_feeder = fluid.DataFeeder(feed_list=feed_list, place=place)
-        num_eval_examples = acc_sum = loss_sum = 0
-        test_reader = data_reader.data_generator(
-            batch_size=batch_size, phase=phase)
-        eval_time_begin = time.time()
-        eval_step = 0
-        total_label, total_infer, total_correct = 0.0, 0.0, 0.0
-        for batch in test_reader():
-            num_batch_examples = len(batch)
-            eval_step += 1
-            np_labels, np_infers, np_lens, _ = exe.run(
-                feed=data_feeder.feed(batch), fetch_list=fetch_list)
-            label_num, infer_num, correct_num = chunk_eval(
-                np_labels, np_infers, np_lens, 7, dev_count)
-
-            total_infer += infer_num
-            total_label += label_num
-            total_correct += correct_num
-
-        precision, recall, f1 = calculate_f1(total_label, total_infer,
-                                             total_correct)
-        eval_time_used = time.time() - eval_time_begin
-        eval_speed = eval_step / eval_time_used
-        logger.info(
-            "[%s evaluation] F1-Score=%f, precision=%f, recall=%f [step/sec: %.2f]"
-            % (phase, f1, precision, recall, eval_speed))
-
-
 def _finetune_cls_task(task, data_reader, feed_list, config=None,
                        do_eval=False):
     main_program = task.main_program()
@@ -287,7 +250,7 @@ def _finetune_cls_task(task, data_reader, feed_list, config=None,
                         exe=exe)
 
                 if do_eval and global_step % config.eval_interval == 0:
-                    eval_loss, eval_acc, eval_perf = evaluate(
+                    eval_loss, eval_acc, eval_perf = evaluate_cls_task(
                         task,
                         data_reader,
                         feed_list,
@@ -313,7 +276,8 @@ def _finetune_cls_task(task, data_reader, feed_list, config=None,
             exe=exe)
 
         if do_eval:
-            evaluate(task, data_reader, feed_list, phase="test", config=config)
+            evaluate_cls_task(
+                task, data_reader, feed_list, phase="test", config=config)
         logger.info("PaddleHub finetune finished.")
 
 
@@ -321,150 +285,10 @@ def finetune_and_eval(task, data_reader, feed_list, config=None):
     if task.task_type == "sequence_labeling":
         _finetune_seq_label_task(
             task, data_reader, feed_list, config, do_eval=True)
+    # if it's image_classification and text classificaiton
     else:
         _finetune_cls_task(task, data_reader, feed_list, config, do_eval=True)
 
 
 def finetune(task, data_reader, feed_list, config=None):
     _finetune_cls_task(task, data_reader, feed_list, config, do_eval=False)
-
-
-def evaluate(task, data_reader, feed_list, phase="test", config=None):
-    logger.info("Evaluation on {} dataset start".format(phase))
-    inference_program = task.inference_program()
-    main_program = task.main_program()
-    loss = task.variable("loss")
-    accuracy = task.variable("accuracy")
-    batch_size = config.batch_size
-    place, dev_count = _get_running_device_info(config)
-    exe = fluid.Executor(place=place)
-    with fluid.program_guard(inference_program):
-        data_feeder = fluid.DataFeeder(feed_list=feed_list, place=place)
-        num_eval_examples = acc_sum = loss_sum = 0
-        test_reader = data_reader.data_generator(
-            batch_size=batch_size, phase=phase)
-        eval_time_begin = time.time()
-        eval_step = 0
-        for batch in test_reader():
-            num_batch_examples = len(batch)
-            eval_step += 1
-            loss_v, accuracy_v = exe.run(
-                feed=data_feeder.feed(batch),
-                fetch_list=[loss.name, accuracy.name])
-            num_eval_examples += num_batch_examples
-            acc_sum += accuracy_v * num_batch_examples
-            loss_sum += loss_v * num_batch_examples
-        eval_time_used = time.time() - eval_time_begin
-
-        avg_loss = loss_sum / num_eval_examples
-        avg_acc = acc_sum / num_eval_examples
-        eval_speed = eval_step / eval_time_used
-    logger.info(
-        "[%s dataset evaluation result] loss=%.5f acc=%.5f [step/sec: %.2f]" %
-        (phase, avg_loss, avg_acc, eval_speed))
-
-    return avg_loss, avg_acc, eval_speed
-
-
-# Sequence label evaluation functions
-def chunk_eval(np_labels, np_infers, np_lens, tag_num, dev_count=1):
-    def extract_bio_chunk(seq):
-        chunks = []
-        cur_chunk = None
-        null_index = tag_num - 1
-        for index in range(len(seq)):
-            tag = seq[index]
-            tag_type = tag // 2
-            tag_pos = tag % 2
-
-            if tag == null_index:
-                if cur_chunk is not None:
-                    chunks.append(cur_chunk)
-                    cur_chunk = None
-                continue
-
-            if tag_pos == 0:
-                if cur_chunk is not None:
-                    chunks.append(cur_chunk)
-                    cur_chunk = {}
-                cur_chunk = {"st": index, "en": index + 1, "type": tag_type}
-
-            else:
-                if cur_chunk is None:
-                    cur_chunk = {"st": index, "en": index + 1, "type": tag_type}
-                    continue
-
-                if cur_chunk["type"] == tag_type:
-                    cur_chunk["en"] = index + 1
-                else:
-                    chunks.append(cur_chunk)
-                    cur_chunk = {"st": index, "en": index + 1, "type": tag_type}
-
-        if cur_chunk is not None:
-            chunks.append(cur_chunk)
-        return chunks
-
-    null_index = tag_num - 1
-    num_label = 0
-    num_infer = 0
-    num_correct = 0
-    labels = np_labels.reshape([-1]).astype(np.int32).tolist()
-    infers = np_infers.reshape([-1]).astype(np.int32).tolist()
-    all_lens = np_lens.reshape([dev_count, -1]).astype(np.int32).tolist()
-
-    base_index = 0
-    for dev_index in range(dev_count):
-        lens = all_lens[dev_index]
-        max_len = 0
-        for l in lens:
-            max_len = max(max_len, l)
-
-        for i in range(len(lens)):
-            seq_st = base_index + i * max_len + 1
-            seq_en = seq_st + (lens[i] - 2)
-            infer_chunks = extract_bio_chunk(infers[seq_st:seq_en])
-            label_chunks = extract_bio_chunk(labels[seq_st:seq_en])
-            num_infer += len(infer_chunks)
-            num_label += len(label_chunks)
-
-            infer_index = 0
-            label_index = 0
-            while label_index < len(label_chunks) \
-                   and infer_index < len(infer_chunks):
-                if infer_chunks[infer_index]["st"] \
-                    < label_chunks[label_index]["st"]:
-                    infer_index += 1
-                elif infer_chunks[infer_index]["st"] \
-                    > label_chunks[label_index]["st"]:
-                    label_index += 1
-                else:
-                    if infer_chunks[infer_index]["en"] \
-                        == label_chunks[label_index]["en"] \
-                        and infer_chunks[infer_index]["type"] \
-                        == label_chunks[label_index]["type"]:
-                        num_correct += 1
-
-                    infer_index += 1
-                    label_index += 1
-
-        base_index += max_len * len(lens)
-
-    return num_label, num_infer, num_correct
-
-
-def calculate_f1(num_label, num_infer, num_correct):
-    if num_infer == 0:
-        precision = 0.0
-    else:
-        precision = num_correct * 1.0 / num_infer
-
-    if num_label == 0:
-        recall = 0.0
-    else:
-        recall = num_correct * 1.0 / num_label
-
-    if num_correct == 0:
-        f1 = 0.0
-    else:
-        f1 = 2 * precision * recall / (precision + recall)
-    return precision, recall, f1

paddlehub/finetune/task.py

@@ -148,7 +148,7 @@ def create_img_classification_task(feature,
     return task
 
 
-def create_seq_labeling_task(feature, labels, seq_len, num_classes=None):
+def create_seq_labeling_task(feature, labels, seq_len, num_classes):
     logits = fluid.layers.fc(
         input=feature,
         size=num_classes,