Release bert source code

.gitignore

+*.pyc

.pre-commit-config.yaml

+-   repo: https://github.com/PaddlePaddle/mirrors-yapf.git
+    sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37
+    hooks:
+    -   id: yapf
+        files: \.py$
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    sha: a11d9314b22d8f8c7556443875b731ef05965464
+    hooks:
+    -   id: check-merge-conflict
+    -   id: check-symlinks
+    -   id: detect-private-key
+        files: (?!.*paddle)^.*$
+    -   id: end-of-file-fixer
+        files: \.md$
+    -   id: trailing-whitespace
+        files: \.md$
+-   repo: https://github.com/Lucas-C/pre-commit-hooks
+    sha: v1.0.1
+    hooks:
+    -   id: forbid-crlf
+        files: \.md$
+    -   id: remove-crlf
+        files: \.md$
+    -   id: forbid-tabs
+        files: \.md$
+    -   id: remove-tabs
+        files: \.md$

.style.yapf

+[style]
+based_on_style = pep8
+column_limit = 80

BERT/batching.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Mask, padding and batching."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+
+def mask(batch_tokens, total_token_num, vocab_size, CLS=1, SEP=2, MASK=3):
+    """
+    Add mask for batch_tokens, return out, mask_label, mask_pos;
+    Note: mask_pos responding the batch_tokens after padded;
+    """
+    max_len = max([len(sent) for sent in batch_tokens])
+    mask_label = []
+    mask_pos = []
+    prob_mask = np.random.rand(total_token_num)
+    # Note: the first token is [CLS], so [low=1]
+    replace_ids = np.random.randint(1, high=vocab_size, size=total_token_num)
+    pre_sent_len = 0
+    prob_index = 0
+    for sent_index, sent in enumerate(batch_tokens):
+        mask_flag = False
+        prob_index += pre_sent_len
+        for token_index, token in enumerate(sent):
+            prob = prob_mask[prob_index + token_index]
+            if prob > 0.15:
+                continue
+            elif 0.03 < prob <= 0.15:
+                # mask
+                if token != SEP and token != CLS:
+                    mask_label.append(sent[token_index])
+                    sent[token_index] = MASK
+                    mask_flag = True
+                    mask_pos.append(sent_index * max_len + token_index)
+            elif 0.015 < prob <= 0.03:
+                # random replace
+                if token != SEP and token != CLS:
+                    mask_label.append(sent[token_index])
+                    sent[token_index] = replace_ids[prob_index + token_index]
+                    mask_flag = True
+                    mask_pos.append(sent_index * max_len + token_index)
+            else:
+                # keep the original token
+                if token != SEP and token != CLS:
+                    mask_label.append(sent[token_index])
+                    mask_pos.append(sent_index * max_len + token_index)
+        pre_sent_len = len(sent)
+
+        # ensure at least mask one word in a sentence
+        while not mask_flag:
+            token_index = int(np.random.randint(1, high=len(sent) - 1, size=1))
+            if sent[token_index] != SEP and sent[token_index] != CLS:
+                mask_label.append(sent[token_index])
+                sent[token_index] = MASK
+                mask_flag = True
+                mask_pos.append(sent_index * max_len + token_index)
+    mask_label = np.array(mask_label).astype("int64").reshape([-1, 1])
+    mask_pos = np.array(mask_pos).astype("int64").reshape([-1, 1])
+    return batch_tokens, mask_label, mask_pos
+
+
+def prepare_batch_data(insts,
+                       total_token_num,
+                       voc_size=0,
+                       pad_id=None,
+                       cls_id=None,
+                       sep_id=None,
+                       mask_id=None,
+                       return_attn_bias=True,
+                       return_max_len=True,
+                       return_num_token=False):
+    """
+    1. generate Tensor of data
+    2. generate Tensor of position
+    3. generate self attention mask, [shape: batch_size *  max_len * max_len]
+    """
+
+    batch_src_ids = [inst[0] for inst in insts]
+    batch_sent_ids = [inst[1] for inst in insts]
+    batch_pos_ids = [inst[2] for inst in insts]
+    labels_list = []
+    # compatible with squad, whose example includes start/end positions, 
+    # or unique id
+
+    for i in range(3, len(insts[0]), 1):
+        labels = [inst[i] for inst in insts]
+        labels = np.array(labels).astype("int64").reshape([-1, 1])
+        labels_list.append(labels)
+
+    # First step: do mask without padding
+    if mask_id >= 0:
+        out, mask_label, mask_pos = mask(
+            batch_src_ids,
+            total_token_num,
+            vocab_size=voc_size,
+            CLS=cls_id,
+            SEP=sep_id,
+            MASK=mask_id)
+    else:
+        out = batch_src_ids
+    # Second step: padding
+    src_id, next_sent_index, self_attn_bias = pad_batch_data(
+        out, pad_idx=pad_id, return_next_sent_pos=True, return_attn_bias=True)
+    pos_id = pad_batch_data(
+        batch_pos_ids, pad_idx=pad_id, return_pos=False, return_attn_bias=False)
+    sent_id = pad_batch_data(
+        batch_sent_ids,
+        pad_idx=pad_id,
+        return_pos=False,
+        return_attn_bias=False)
+
+    if mask_id >= 0:
+        return_list = [src_id, pos_id, sent_id, self_attn_bias, mask_label, mask_pos] \
+                      + labels_list + [next_sent_index]
+    else:
+        return_list = [src_id, pos_id, sent_id, self_attn_bias] + labels_list \
+                     + [next_sent_index]
+
+    return return_list if len(return_list) > 1 else return_list[0]
+
+
+def pad_batch_data(insts,
+                   pad_idx=0,
+                   return_pos=False,
+                   return_next_sent_pos=False,
+                   return_attn_bias=False,
+                   return_max_len=False,
+                   return_num_token=False):
+    """
+    Pad the instances to the max sequence length in batch, and generate the
+    corresponding position data and attention bias.
+    """
+    return_list = []
+    max_len = max(len(inst) for inst in insts)
+    # Any token included in dict can be used to pad, since the paddings' loss
+    # will be masked out by weights and make no effect on parameter gradients.
+
+    inst_data = np.array(
+        [inst + list([pad_idx] * (max_len - len(inst))) for inst in insts])
+    return_list += [inst_data.astype("int64").reshape([-1, max_len, 1])]
+
+    # next_sent_pos for extract first token embedding of each sentence
+    if return_next_sent_pos:
+        batch_size = inst_data.shape[0]
+        max_seq_len = inst_data.shape[1]
+        next_sent_index = np.array(
+            range(0, batch_size * max_seq_len, max_seq_len)).astype(
+                "int64").reshape(-1, 1)
+        return_list += [next_sent_index]
+
+    # position data
+    if return_pos:
+        inst_pos = np.array([
+            list(range(0, len(inst))) + [pad_idx] * (max_len - len(inst))
+            for inst in insts
+        ])
+
+        return_list += [inst_pos.astype("int64").reshape([-1, max_len, 1])]
+
+    if return_attn_bias:
+        # This is used to avoid attention on paddings.
+        slf_attn_bias_data = np.array([[0] * len(inst) + [-1e9] *
+                                       (max_len - len(inst)) for inst in insts])
+        slf_attn_bias_data = np.tile(
+            slf_attn_bias_data.reshape([-1, 1, max_len]), [1, max_len, 1])
+        return_list += [slf_attn_bias_data.astype("float32")]
+
+    if return_max_len:
+        return_list += [max_len]
+
+    if return_num_token:
+        num_token = 0
+        for inst in insts:
+            num_token += len(inst)
+        return_list += [num_token]
+
+    return return_list if len(return_list) > 1 else return_list[0]
+
+
+if __name__ == "__main__":
+    pass

BERT/config/bert_config.json

+{
+  "attention_probs_dropout_prob": 0.1, 
+  "directionality": "bidi", 
+  "hidden_act": "gelu", 
+  "hidden_dropout_prob": 0.1, 
+  "hidden_size": 768, 
+  "initializer_range": 0.02, 
+  "intermediate_size": 3072, 
+  "max_position_embeddings": 512, 
+  "num_attention_heads": 12, 
+  "num_hidden_layers": 12, 
+  "pooler_fc_size": 768, 
+  "pooler_num_attention_heads": 12, 
+  "pooler_num_fc_layers": 3, 
+  "pooler_size_per_head": 128, 
+  "pooler_type": "first_token_transform", 
+  "type_vocab_size": 2, 
+  "vocab_size": 21128
+}

BERT/convert_params.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Convert Google official BERT models to Fluid parameters."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import argparse
+import collections
+from args import print_arguments
+import tensorflow as tf
+import paddle.fluid as fluid
+from tensorflow.python import pywrap_tensorflow
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument(
+        "--init_tf_checkpoint",
+        type=str,
+        required=True,
+        help="Initial TF checkpoint (a pre-trained BERT model).")
+
+    parser.add_argument(
+        "--fluid_params_dir",
+        type=str,
+        required=True,
+        help="The directory to store converted Fluid parameters.")
+    args = parser.parse_args()
+    return args
+
+
+def parse(init_checkpoint):
+    tf_fluid_param_name_map = collections.OrderedDict()
+    tf_param_name_shape_map = collections.OrderedDict()
+
+    init_vars = tf.train.list_variables(init_checkpoint)
+    for (var_name, var_shape) in init_vars:
+        fluid_param_name = ''
+        if var_name.startswith('bert/'):
+            key = var_name[5:]
+            if (key.startswith('embeddings/')):
+                if (key.endswith('LayerNorm/gamma')):
+                    fluid_param_name = 'pre_encoder_layer_norm_scale'
+                elif (key.endswith('LayerNorm/beta')):
+                    fluid_param_name = 'pre_encoder_layer_norm_bias'
+                elif (key.endswith('position_embeddings')):
+                    fluid_param_name = 'pos_embedding'
+                elif (key.endswith('word_embeddings')):
+                    fluid_param_name = 'word_embedding'
+                elif (key.endswith('token_type_embeddings')):
+                    fluid_param_name = 'sent_embedding'
+                else:
+                    print("ignored param: %s" % var_name)
+            elif (key.startswith('encoder/')):
+                key = key[8:]
+                layer_num = int(key[key.find('_') + 1:key.find('/')])
+                suffix = "encoder_layer_" + str(layer_num)
+                if key.endswith('attention/output/LayerNorm/beta'):
+                    fluid_param_name = suffix + '_post_att_layer_norm_bias'
+                elif key.endswith('attention/output/LayerNorm/gamma'):
+                    fluid_param_name = suffix + '_post_att_layer_norm_scale'
+                elif key.endswith('attention/output/dense/bias'):
+                    fluid_param_name = suffix + '_multi_head_att_output_fc.b_0'
+                elif key.endswith('attention/output/dense/kernel'):
+                    fluid_param_name = suffix + '_multi_head_att_output_fc.w_0'
+                elif key.endswith('attention/self/key/bias'):
+                    fluid_param_name = suffix + '_multi_head_att_key_fc.b_0'
+                elif key.endswith('attention/self/key/kernel'):
+                    fluid_param_name = suffix + '_multi_head_att_key_fc.w_0'
+                elif key.endswith('attention/self/query/bias'):
+                    fluid_param_name = suffix + '_multi_head_att_query_fc.b_0'
+                elif key.endswith('attention/self/query/kernel'):
+                    fluid_param_name = suffix + '_multi_head_att_query_fc.w_0'
+                elif key.endswith('attention/self/value/bias'):
+                    fluid_param_name = suffix + '_multi_head_att_value_fc.b_0'
+                elif key.endswith('attention/self/value/kernel'):
+                    fluid_param_name = suffix + '_multi_head_att_value_fc.w_0'
+                elif key.endswith('intermediate/dense/bias'):
+                    fluid_param_name = suffix + '_ffn_fc_0.b_0'
+                elif key.endswith('intermediate/dense/kernel'):
+                    fluid_param_name = suffix + '_ffn_fc_0.w_0'
+                elif key.endswith('output/LayerNorm/beta'):
+                    fluid_param_name = suffix + '_post_ffn_layer_norm_bias'
+                elif key.endswith('output/LayerNorm/gamma'):
+                    fluid_param_name = suffix + '_post_ffn_layer_norm_scale'
+                elif key.endswith('output/dense/bias'):
+                    fluid_param_name = suffix + '_ffn_fc_1.b_0'
+                elif key.endswith('output/dense/kernel'):
+                    fluid_param_name = suffix + '_ffn_fc_1.w_0'
+                else:
+                    print("ignored param: %s" % var_name)
+            elif (key.startswith('pooler/')):
+                if key.endswith('dense/bias'):
+                    fluid_param_name = 'pooled_fc.b_0'
+                elif key.endswith('dense/kernel'):
+                    fluid_param_name = 'pooled_fc.w_0'
+                else:
+                    print("ignored param: %s" % var_name)
+            else:
+                print("ignored param: %s" % var_name)
+
+        elif var_name.startswith('cls/'):
+            if var_name == 'cls/predictions/output_bias':
+                fluid_param_name = 'mask_lm_out_fc.b_0'
+            elif var_name == 'cls/predictions/transform/LayerNorm/beta':
+                fluid_param_name = 'mask_lm_trans_layer_norm_bias'
+            elif var_name == 'cls/predictions/transform/LayerNorm/gamma':
+                fluid_param_name = 'mask_lm_trans_layer_norm_scale'
+            elif var_name == 'cls/predictions/transform/dense/bias':
+                fluid_param_name = 'mask_lm_trans_fc.b_0'
+            elif var_name == 'cls/predictions/transform/dense/kernel':
+                fluid_param_name = 'mask_lm_trans_fc.w_0'
+            elif var_name == 'cls/seq_relationship/output_bias':
+                fluid_param_name = 'next_sent_fc.b_0'
+            elif var_name == 'cls/seq_relationship/output_weights':
+                fluid_param_name = 'next_sent_fc.w_0'
+            elif var_name == 'cls/squad/output_weights':
+                fluid_param_name = 'cls_squad_out_w'
+            elif var_name == 'cls/squad/output_bias':
+                fluid_param_name = 'cls_squad_out_b'
+            else:
+                print("ignored param: %s" % var_name)
+        else:
+            print("ignored param: %s" % var_name)
+
+        if fluid_param_name is not '':
+            tf_fluid_param_name_map[var_name] = fluid_param_name
+            tf_param_name_shape_map[var_name] = var_shape
+            fluid_param_name = ''
+
+    return tf_fluid_param_name_map, tf_param_name_shape_map
+
+
+def convert(args):
+    tf_fluid_param_name_map, tf_param_name_shape_map = parse(
+        args.init_tf_checkpoint)
+    program = fluid.Program()
+    global_block = program.global_block()
+    for param in tf_fluid_param_name_map:
+        global_block.create_parameter(
+            name=tf_fluid_param_name_map[param],
+            shape=tf_param_name_shape_map[param],
+            dtype='float32',
+            initializer=fluid.initializer.Constant(value=0.0))
+
+    place = fluid.core.CPUPlace()
+    exe = fluid.Executor(place)
+    exe.run(program)
+
+    print('---------------------- Converted Parameters -----------------------')
+    print('###### [TF param name] --> [Fluid param name]  [param shape] ######')
+    print('-------------------------------------------------------------------')
+
+    reader = pywrap_tensorflow.NewCheckpointReader(args.init_tf_checkpoint)
+    for param in tf_fluid_param_name_map:
+        value = reader.get_tensor(param)
+        if param == 'cls/seq_relationship/output_weights':
+            value = np.transpose(value)
+        if param == 'cls/squad/output_weights':
+            value = np.transpose(value)
+        fluid.global_scope().find_var(tf_fluid_param_name_map[
+            param]).get_tensor().set(value, place)
+        print(param, ' --> ', tf_fluid_param_name_map[param], '  ', value.shape)
+
+    fluid.io.save_params(exe, args.fluid_params_dir, main_program=program)
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    print_arguments(args)
+    convert(args)

BERT/inference/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 3.2)
+PROJECT(inference_demo)
+SET(CMAKE_C_COMPILER gcc-4.8)
+SET(CMAKE_CXX_COMPILER g++-4.8)
+ADD_COMPILE_OPTIONS(-std=c++11)
+
+SET(FLUID_INFER_LIB fluid_inference)
+SET(FLUID_INC_PATH ${FLUID_INFER_LIB}/paddle/include)
+SET(FLUID_LIB_PATH ${FLUID_INFER_LIB}/paddle/lib)
+
+SET(GLOG_INC_PATH ${FLUID_INFER_LIB}/third_party/install/glog/include)
+SET(GLOG_LIB_PATH ${FLUID_INFER_LIB}/third_party/install/glog/lib)
+
+SET(GFLAGS_INC_PATH ${FLUID_INFER_LIB}/third_party/install/gflags/include)
+SET(GFLAGS_LIB_PATH ${FLUID_INFER_LIB}/third_party/install/gflags/lib)
+SET(MKLDNN_LIB_PATH ${FLUID_INFER_LIB}/third_party/install/mkldnn/lib)
+SET(MKLML_LIB_PATH ${FLUID_INFER_LIB}/third_party/install/mklml/lib)
+
+INCLUDE_DIRECTORIES(${FLUID_INC_PATH})
+INCLUDE_DIRECTORIES(${GLOG_INC_PATH})
+INCLUDE_DIRECTORIES(${GFLAGS_INC_PATH})
+
+LINK_DIRECTORIES(${FLUID_LIB_PATH})
+LINK_DIRECTORIES(${GLOG_LIB_PATH})
+LINK_DIRECTORIES(${GFLAGS_LIB_PATH})
+LINK_DIRECTORIES(${MKLML_LIB_PATH})
+LINK_DIRECTORIES(${MKLDNN_LIB_PATH})
+
+ADD_EXECUTABLE(inference inference.cc)
+TARGET_LINK_LIBRARIES(inference dl paddle_fluid glog gflags pthread)
+

BERT/inference/README.md

+# BERT模型inference demo
+
+## 数据预处理
+实际应用场景中，模型部署之后用户还需要编写对应的程序对输入进行处理，然后把得到的数据传给模型进行预测。这里为了演示的需要，用 `gen_demo_data.py` 来进行数据处理，包括 tokenization，batching，numericalization，并且把处理后的数据输出为文本文件。使用方法如下：
+
+``` bash
+TASK_NAME="xnli"
+DATA_PATH=/path/to/xnli/data/
+BERT_BASE_PATH=/path/to/bert/pretrained/model/
+python gen_demo_data.py \
+    --task_name ${TASK_NAME} \
+    --data_path ${DATA_PATH} \
+    --vocab_path "${BERT_BASE_PATH}/vocab.txt" \
+    --batch_size 4096 \
+    --in_tokens \
+    > data.txt
+```
+
+**生成的数据格式**
+
+生成的数据一行代表一个 `batch`, 包含五个字段
+
+```text
+src_id, pos_id, segment_id, self_attention_bias, next_segment_index
+```
+
+字段之间按照分号(;)分隔，其中各字段内部 `shape` 和 `data` 按照冒号(:)分隔，`shape` 和 `data` 内部按空格分隔，`self_attention_bias` 为 FLOAT32 类型，其余字段为 INT64 类型。
+
+## 编译和运行
+
+为了编译 inference demo，`c++` 编译器需要支持 `C++11` 标准。
+
+首先下载对应的 [Fluid_inference库](http://paddlepaddle.org/documentation/docs/zh/1.2/advanced_usage/deploy/inference/build_and_install_lib_cn.html) , 根据使用的 paddle 的版本和配置状况 (是否使用 avx, mkl, 以及 cuda, cudnn 版本) 选择下载对应的版本，并解压至 `inference` 目录，会得到 `fluid_inference` 文件夹。
+
+设置运行相关的环境变量(以 `cpu_avx_mkl` 版本为例)
+
+``` bash
+LD_LIBRARY_PATH=fluid_inference/paddle/lib/:$LD_LIBRARY_PATH
+LD_LIBRARY_PATH=fluid_inference/third_party/install/mklml/lib:$LD_LIBRARY_PATH
+LD_LIBRARY_PATH=fluid_inference/third_party/install/mkldnn/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH
+```
+
+编译 demo
+
+``` bash
+mkdir build
+cd build
+cmake
+make
+```
+
+这会在 `build` 目录下生成运行 `inference` 可执行文件。
+
+运行 demo
+
+```bash
+./inference --logtostderr \
+    --model_dir $MODEL_PATH \
+    --data $DATA_PATH \
+    --repeat $REPEAT_NUM
+```
+
+## 性能测试
+
+E5-2680 CPU，单线程下196样本性能如下
+
+| repeat | 单样本耗时(ms) |
+| -----  | -----          |
+| 10     | 264.696        |
+| 50     | 264.84         |

BERT/inference/gen_demo_data.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+sys.path.append("..")
+from reader import cls
+
+
+def main():
+    args = parse_args()
+    task_name = args.task_name.lower()
+    processors = {
+        'xnli': cls.XnliProcessor,
+        'cola': cls.ColaProcessor,
+        'mrpc': cls.MrpcProcessor,
+        'mnli': cls.MnliProcessor,
+    }
+
+    processor = processors[task_name](data_dir=args.data_path,
+                                      vocab_path=args.vocab_path,
+                                      max_seq_len=args.max_seq_len,
+                                      do_lower_case=args.do_lower_case,
+                                      in_tokens=args.in_tokens,
+                                      random_seed=args.random_seed)
+    example = processor.get_test_examples(args.data_path)[0]
+    gen = processor.data_generator(
+        args.batch_size, phase='test', epoch=1, shuffle=False)()
+
+    for i, data in enumerate(gen):
+        data = data[:4] + [data[5]]
+        sample = []
+        for field in data:
+            shape_str = ' '.join(map(str, field.shape))
+            data_str = ' '.join(map(str, field.reshape(-1).tolist()))
+            sample.append(shape_str + ':' + data_str)
+        print(';'.join(sample))
+
+
+def str2bool(v):
+    # because argparse does not support to parse "true, False" as python
+    # boolean directly
+    return v.lower() in ("true", "t", "1")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(prog="bert data prepare")
+    parser.add_argument(
+        "--task_name",
+        type=str,
+        default='xnli',
+        choices=["xnli", "mnli", "cola", "mrpc"],
+        help="task name, used to specify data preprocessor")
+    parser.add_argument(
+        "--batch_size",
+        type=int,
+        default=4096,
+        help="batch size, see also --in_tokens")
+    parser.add_argument(
+        "--in_tokens",
+        action='store_true',
+        help="if set, batch_size means token number in a batch, otherwise "
+        "it means example number in a batch")
+    parser.add_argument(
+        '--do_lower_case',
+        type=str2bool,
+        default=True,
+        choices=[True, False],
+        help="Whether to lower case the input text. Should be True for uncased "
+        "models and False for cased models.")
+    parser.add_argument("--vocab_path", type=str, help="path of vocabulary")
+    parser.add_argument("--data_path", type=str, help="path of data to process")
+    parser.add_argument(
+        "--max_seq_len", type=int, default=128, help="max sequence length")
+    parser.add_argument(
+        "--random_seed", type=int, default=0, help="random seed")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main()

BERT/inference/inference.cc

+// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <paddle_inference_api.h>
+#include <chrono>
+#include <fstream>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <vector>
+
+DEFINE_string(model_dir, "", "model directory");
+DEFINE_string(data, "", "input data path");
+DEFINE_int32(repeat, 1, "repeat");
+
+template <typename T>
+void GetValueFromStream(std::stringstream *ss, T *t) {
+  (*ss) >> (*t);
+}
+
+template <>
+void GetValueFromStream<std::string>(std::stringstream *ss, std::string *t) {
+  *t = ss->str();
+}
+
+// Split string to vector
+template <typename T>
+void Split(const std::string &line, char sep, std::vector<T> *v) {
+  std::stringstream ss;
+  T t;
+  for (auto c : line) {
+    if (c != sep) {
+      ss << c;
+    } else {
+      GetValueFromStream<T>(&ss, &t);
+      v->push_back(std::move(t));
+      ss.str({});
+      ss.clear();
+    }
+  }
+
+  if (!ss.str().empty()) {
+    GetValueFromStream<T>(&ss, &t);
+    v->push_back(std::move(t));
+    ss.str({});
+    ss.clear();
+  }
+}
+
+template <typename T>
+constexpr paddle::PaddleDType GetPaddleDType();
+
+template <>
+constexpr paddle::PaddleDType GetPaddleDType<int64_t>() {
+  return paddle::PaddleDType::INT64;
+}
+
+template <>
+constexpr paddle::PaddleDType GetPaddleDType<float>() {
+  return paddle::PaddleDType::FLOAT32;
+}
+
+// Parse tensor from string
+template <typename T>
+bool ParseTensor(const std::string &field, paddle::PaddleTensor *tensor) {
+  std::vector<std::string> data;
+  Split(field, ':', &data);
+  if (data.size() < 2) return false;
+
+  std::string shape_str = data[0];
+
+  std::vector<int> shape;
+  Split(shape_str, ' ', &shape);
+
+  std::string mat_str = data[1];
+
+  std::vector<T> mat;
+  Split(mat_str, ' ', &mat);
+
+  tensor->shape = shape;
+  auto size =
+      std::accumulate(shape.begin(), shape.end(), 1, std::multiplies<int>()) *
+      sizeof(T);
+  tensor->data.Resize(size);
+  std::copy(mat.begin(), mat.end(), static_cast<T *>(tensor->data.data()));
+  tensor->dtype = GetPaddleDType<T>();
+
+  return true;
+}
+
+// Parse input tensors from string
+bool ParseLine(const std::string &line,
+               std::vector<paddle::PaddleTensor> *tensors) {
+  std::vector<std::string> fields;
+  Split(line, ';', &fields);
+
+  if (fields.size() < 5) return false;
+
+  tensors->clear();
+  tensors->reserve(5);
+
+  int i = 0;
+  // src_id
+  paddle::PaddleTensor src_id;
+  ParseTensor<int64_t>(fields[i++], &src_id);
+  tensors->push_back(src_id);
+
+  // pos_id
+  paddle::PaddleTensor pos_id;
+  ParseTensor<int64_t>(fields[i++], &pos_id);
+  tensors->push_back(pos_id);
+
+  // segment_id
+  paddle::PaddleTensor segment_id;
+  ParseTensor<int64_t>(fields[i++], &segment_id);
+  tensors->push_back(segment_id);
+
+  // self_attention_bias
+  paddle::PaddleTensor self_attention_bias;
+  ParseTensor<float>(fields[i++], &self_attention_bias);
+  tensors->push_back(self_attention_bias);
+
+  // next_segment_index
+  paddle::PaddleTensor next_segment_index;
+  ParseTensor<int64_t>(fields[i++], &next_segment_index);
+  tensors->push_back(next_segment_index);
+
+  return true;
+}
+
+// Print outputs to log
+void PrintOutputs(const std::vector<paddle::PaddleTensor> &outputs) {
+  LOG(INFO) << "example_id\tcontradiction\tentailment\tneutral";
+
+  for (size_t i = 0; i < outputs.front().data.length() / sizeof(float); i += 3) {
+    LOG(INFO) << (i / 3) << "\t"
+              << static_cast<float *>(outputs.front().data.data())[i] << "\t"
+              << static_cast<float *>(outputs.front().data.data())[i + 1]
+              << "\t"
+              << static_cast<float *>(outputs.front().data.data())[i + 2];
+  }
+}
+
+bool LoadInputData(std::vector<std::vector<paddle::PaddleTensor>> *inputs) {
+  if (FLAGS_data.empty()) {
+    LOG(ERROR) << "please set input data path";
+    return false;
+  }
+
+  std::ifstream fin(FLAGS_data);
+  std::string line;
+
+  int lineno = 0;
+  while (std::getline(fin, line)) {
+    std::vector<paddle::PaddleTensor> feed_data;
+    if (!ParseLine(line, &feed_data)) {
+      LOG(ERROR) << "Parse line[" << lineno << "] error!";
+    } else {
+      inputs->push_back(std::move(feed_data));
+    }
+  }
+
+  return true;
+}
+
+// Bert inference demo
+// Options:
+//     --model_dir: bert model file directory
+//     --data: data path
+//     --repeat: repeat num
+int main(int argc, char *argv[]) {
+  google::InitGoogleLogging(*argv);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  if (FLAGS_model_dir.empty()) {
+    LOG(ERROR) << "please set model dir";
+    return -1;
+  }
+
+  paddle::NativeConfig config;
+  config.model_dir = FLAGS_model_dir;
+  config.use_gpu = false;
+
+  auto predictor = CreatePaddlePredictor(config);
+
+  std::vector<std::vector<paddle::PaddleTensor>> inputs;
+  if (!LoadInputData(&inputs)) {
+    LOG(ERROR) << "load input data error!";
+    return -1;
+  }
+
+  std::vector<paddle::PaddleTensor> fetch;
+  int total_time{0};
+  // auto predict_timer = []()
+  int num_samples{0};
+  for (int i = 0; i < FLAGS_repeat; i++) {
+    for (auto feed : inputs) {
+      auto start = std::chrono::system_clock::now();
+      predictor->Run(feed, &fetch);
+      auto end = std::chrono::system_clock::now();
+      if (!fetch.empty()) {
+        total_time +=
+            std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
+                .count();
+        num_samples += fetch.front().data.length() / 3;
+      }
+    }
+  }
+
+  auto per_sample_ms =
+      static_cast<float>(total_time) / num_samples;
+  LOG(INFO) << "Run " << num_samples
+            << " samples, average latency: " << per_sample_ms
+            << "ms per sample.";
+
+  return 0;
+}

BERT/model/bert.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""BERT model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six
+import json
+import numpy as np
+import paddle.fluid as fluid
+from model.transformer_encoder import encoder, pre_process_layer
+
+
+class BertConfig(object):
+    def __init__(self, config_path):
+        self._config_dict = self._parse(config_path)
+
+    def _parse(self, config_path):
+        try:
+            with open(config_path) as json_file:
+                config_dict = json.load(json_file)
+        except Exception:
+            raise IOError("Error in parsing bert model config file '%s'" %
+                          config_path)
+        else:
+            return config_dict
+
+    def __getitem__(self, key):
+        return self._config_dict[key]
+
+    def print_config(self):
+        for arg, value in sorted(six.iteritems(self._config_dict)):
+            print('%s: %s' % (arg, value))
+        print('------------------------------------------------')
+
+
+class BertModel(object):
+    def __init__(self,
+                 src_ids,
+                 position_ids,
+                 sentence_ids,
+                 self_attn_mask,
+                 config,
+                 weight_sharing=True,
+                 use_fp16=False):
+
+        self._emb_size = config['hidden_size']
+        self._n_layer = config['num_hidden_layers']
+        self._n_head = config['num_attention_heads']
+        self._voc_size = config['vocab_size']
+        self._max_position_seq_len = config['max_position_embeddings']
+        self._sent_types = config['type_vocab_size']
+        self._hidden_act = config['hidden_act']
+        self._prepostprocess_dropout = config['hidden_dropout_prob']
+        self._attention_dropout = config['attention_probs_dropout_prob']
+        self._weight_sharing = weight_sharing
+
+        self._word_emb_name = "word_embedding"
+        self._pos_emb_name = "pos_embedding"
+        self._sent_emb_name = "sent_embedding"
+        self._dtype = "float16" if use_fp16 else "float32"
+
+        # Initialize all weigths by truncated normal initializer, and all biases 
+        # will be initialized by constant zero by default.
+        self._param_initializer = fluid.initializer.TruncatedNormal(
+            scale=config['initializer_range'])
+
+        self._build_model(src_ids, position_ids, sentence_ids, self_attn_mask)
+
+    def _build_model(self, src_ids, position_ids, sentence_ids, self_attn_mask):
+        # padding id in vocabulary must be set to 0
+        emb_out = fluid.layers.embedding(
+            input=src_ids,
+            size=[self._voc_size, self._emb_size],
+            dtype=self._dtype,
+            param_attr=fluid.ParamAttr(
+                name=self._word_emb_name, initializer=self._param_initializer),
+            is_sparse=False)
+        position_emb_out = fluid.layers.embedding(
+            input=position_ids,
+            size=[self._max_position_seq_len, self._emb_size],
+            dtype=self._dtype,
+            param_attr=fluid.ParamAttr(
+                name=self._pos_emb_name, initializer=self._param_initializer))
+
+        sent_emb_out = fluid.layers.embedding(
+            sentence_ids,
+            size=[self._sent_types, self._emb_size],
+            dtype=self._dtype,
+            param_attr=fluid.ParamAttr(
+                name=self._sent_emb_name, initializer=self._param_initializer))
+
+        emb_out = emb_out + position_emb_out
+        emb_out = emb_out + sent_emb_out
+
+        emb_out = pre_process_layer(
+            emb_out, 'nd', self._prepostprocess_dropout, name='pre_encoder')
+
+        if self._dtype is "float16":
+            self_attn_mask = fluid.layers.cast(
+                x=self_attn_mask, dtype=self._dtype)
+
+        n_head_self_attn_mask = fluid.layers.stack(
+            x=[self_attn_mask] * self._n_head, axis=1)
+        n_head_self_attn_mask.stop_gradient = True
+
+        self._enc_out = encoder(
+            enc_input=emb_out,
+            attn_bias=n_head_self_attn_mask,
+            n_layer=self._n_layer,
+            n_head=self._n_head,
+            d_key=self._emb_size // self._n_head,
+            d_value=self._emb_size // self._n_head,
+            d_model=self._emb_size,
+            d_inner_hid=self._emb_size * 4,
+            prepostprocess_dropout=self._prepostprocess_dropout,
+            attention_dropout=self._attention_dropout,
+            relu_dropout=0,
+            hidden_act=self._hidden_act,
+            preprocess_cmd="",
+            postprocess_cmd="dan",
+            param_initializer=self._param_initializer,
+            name='encoder')
+
+    def get_sequence_output(self):
+        return self._enc_out
+
+    def get_pooled_output(self, next_sent_index):
+        """Get the first feature of each sequence for classification"""
+        self._reshaped_emb_out = fluid.layers.reshape(
+            x=self._enc_out, shape=[-1, self._emb_size], inplace=True)
+        next_sent_index = fluid.layers.cast(x=next_sent_index, dtype='int32')
+        next_sent_feat = fluid.layers.gather(
+            input=self._reshaped_emb_out, index=next_sent_index)
+        next_sent_feat = fluid.layers.fc(
+            input=next_sent_feat,
+            size=self._emb_size,
+            act="tanh",
+            param_attr=fluid.ParamAttr(
+                name="pooled_fc.w_0", initializer=self._param_initializer),
+            bias_attr="pooled_fc.b_0")
+        return next_sent_feat
+
+    def get_pretraining_output(self, mask_label, mask_pos, labels,
+                               next_sent_index):
+        """Get the loss & accuracy for pretraining"""
+
+        mask_pos = fluid.layers.cast(x=mask_pos, dtype='int32')
+
+        # extract the first token feature in each sentence
+        next_sent_feat = self.get_pooled_output(next_sent_index)
+        # extract masked tokens' feature
+        mask_feat = fluid.layers.gather(
+            input=self._reshaped_emb_out, index=mask_pos)
+
+        # transform: fc
+        mask_trans_feat = fluid.layers.fc(
+            input=mask_feat,
+            size=self._emb_size,
+            act=self._hidden_act,
+            param_attr=fluid.ParamAttr(
+                name='mask_lm_trans_fc.w_0',
+                initializer=self._param_initializer),
+            bias_attr=fluid.ParamAttr(name='mask_lm_trans_fc.b_0'))
+        # transform: layer norm 
+        mask_trans_feat = pre_process_layer(
+            mask_trans_feat, 'n', name='mask_lm_trans')
+
+        mask_lm_out_bias_attr = fluid.ParamAttr(
+            name="mask_lm_out_fc.b_0",
+            initializer=fluid.initializer.Constant(value=0.0))
+        if self._weight_sharing:
+            fc_out = fluid.layers.matmul(
+                x=mask_trans_feat,
+                y=fluid.default_main_program().global_block().var(
+                    self._word_emb_name),
+                transpose_y=True)
+            fc_out += fluid.layers.create_parameter(
+                shape=[self._voc_size],
+                dtype=self._dtype,
+                attr=mask_lm_out_bias_attr,
+                is_bias=True)
+
+        else:
+            fc_out = fluid.layers.fc(input=mask_trans_feat,
+                                     size=self._voc_size,
+                                     param_attr=fluid.ParamAttr(
+                                         name="mask_lm_out_fc.w_0",
+                                         initializer=self._param_initializer),
+                                     bias_attr=mask_lm_out_bias_attr)
+
+        mask_lm_loss = fluid.layers.softmax_with_cross_entropy(
+            logits=fc_out, label=mask_label)
+        mean_mask_lm_loss = fluid.layers.mean(mask_lm_loss)
+
+        next_sent_fc_out = fluid.layers.fc(
+            input=next_sent_feat,
+            size=2,
+            param_attr=fluid.ParamAttr(
+                name="next_sent_fc.w_0", initializer=self._param_initializer),
+            bias_attr="next_sent_fc.b_0")
+
+        next_sent_loss, next_sent_softmax = fluid.layers.softmax_with_cross_entropy(
+            logits=next_sent_fc_out, label=labels, return_softmax=True)
+
+        next_sent_acc = fluid.layers.accuracy(
+            input=next_sent_softmax, label=labels)
+
+        mean_next_sent_loss = fluid.layers.mean(next_sent_loss)
+
+        loss = mean_next_sent_loss + mean_mask_lm_loss
+        return next_sent_acc, mean_mask_lm_loss, loss

BERT/model/classifier.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Model for classifier."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import paddle.fluid as fluid
+
+from model.bert import BertModel
+
+
+def create_model(args,
+                 pyreader_name,
+                 bert_config,
+                 num_labels,
+                 is_prediction=False):
+    pyreader = fluid.layers.py_reader(
+        capacity=50,
+        shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1],
+                [-1, args.max_seq_len, 1],
+                [-1, args.max_seq_len, args.max_seq_len], [-1, 1], [-1, 1]],
+        dtypes=['int64', 'int64', 'int64', 'float', 'int64', 'int64'],
+        lod_levels=[0, 0, 0, 0, 0, 0],
+        name=pyreader_name,
+        use_double_buffer=True)
+
+    (src_ids, pos_ids, sent_ids, self_attn_mask, labels,
+     next_sent_index) = fluid.layers.read_file(pyreader)
+
+    bert = BertModel(
+        src_ids=src_ids,
+        position_ids=pos_ids,
+        sentence_ids=sent_ids,
+        self_attn_mask=self_attn_mask,
+        config=bert_config,
+        use_fp16=args.use_fp16)
+
+    cls_feats = bert.get_pooled_output(next_sent_index)
+    cls_feats = fluid.layers.dropout(
+        x=cls_feats,
+        dropout_prob=0.1,
+        dropout_implementation="upscale_in_train")
+    logits = fluid.layers.fc(
+        input=cls_feats,
+        size=num_labels,
+        param_attr=fluid.ParamAttr(
+            name="cls_out_w",
+            initializer=fluid.initializer.TruncatedNormal(scale=0.02)),
+        bias_attr=fluid.ParamAttr(
+            name="cls_out_b", initializer=fluid.initializer.Constant(0.)))
+
+    if is_prediction:
+        probs = fluid.layers.softmax(logits)
+        feed_targets_name = [
+            src_ids.name, pos_ids.name, sent_ids.name, self_attn_mask.name,
+            next_sent_index.name
+        ]
+        return pyreader, probs, feed_targets_name
+
+    ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
+        logits=logits, label=labels, return_softmax=True)
+    loss = fluid.layers.mean(x=ce_loss)
+
+    if args.use_fp16 and args.loss_scaling > 1.0:
+        loss *= args.loss_scaling
+
+    num_seqs = fluid.layers.create_tensor(dtype='int64')
+    accuracy = fluid.layers.accuracy(input=probs, label=labels, total=num_seqs)
+
+    return pyreader, loss, probs, accuracy, num_seqs

BERT/model/transformer_encoder.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Transformer encoder."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from functools import partial
+import numpy as np
+
+import paddle.fluid as fluid
+import paddle.fluid.layers as layers
+
+
+def multi_head_attention(queries,
+                         keys,
+                         values,
+                         attn_bias,
+                         d_key,
+                         d_value,
+                         d_model,
+                         n_head=1,
+                         dropout_rate=0.,
+                         cache=None,
+                         param_initializer=None,
+                         name='multi_head_att'):
+    """
+    Multi-Head Attention. Note that attn_bias is added to the logit before
+    computing softmax activiation to mask certain selected positions so that
+    they will not considered in attention weights.
+    """
+    keys = queries if keys is None else keys
+    values = keys if values is None else values
+
+    if not (len(queries.shape) == len(keys.shape) == len(values.shape) == 3):
+        raise ValueError(
+            "Inputs: quries, keys and values should all be 3-D tensors.")
+
+    def __compute_qkv(queries, keys, values, n_head, d_key, d_value):
+        """
+        Add linear projection to queries, keys, and values.
+        """
+        q = layers.fc(input=queries,
+                      size=d_key * n_head,
+                      num_flatten_dims=2,
+                      param_attr=fluid.ParamAttr(
+                          name=name + '_query_fc.w_0',
+                          initializer=param_initializer),
+                      bias_attr=name + '_query_fc.b_0')
+        k = layers.fc(input=keys,
+                      size=d_key * n_head,
+                      num_flatten_dims=2,
+                      param_attr=fluid.ParamAttr(
+                          name=name + '_key_fc.w_0',
+                          initializer=param_initializer),
+                      bias_attr=name + '_key_fc.b_0')
+        v = layers.fc(input=values,
+                      size=d_value * n_head,
+                      num_flatten_dims=2,
+                      param_attr=fluid.ParamAttr(
+                          name=name + '_value_fc.w_0',
+                          initializer=param_initializer),
+                      bias_attr=name + '_value_fc.b_0')
+        return q, k, v
+
+    def __split_heads(x, n_head):
+        """
+        Reshape the last dimension of inpunt tensor x so that it becomes two
+        dimensions and then transpose. Specifically, input a tensor with shape
+        [bs, max_sequence_length, n_head * hidden_dim] then output a tensor
+        with shape [bs, n_head, max_sequence_length, hidden_dim].
+        """
+        hidden_size = x.shape[-1]
+        # The value 0 in shape attr means copying the corresponding dimension
+        # size of the input as the output dimension size.
+        reshaped = layers.reshape(
+            x=x, shape=[0, 0, n_head, hidden_size // n_head], inplace=True)
+
+        # permuate the dimensions into:
+        # [batch_size, n_head, max_sequence_len, hidden_size_per_head]
+        return layers.transpose(x=reshaped, perm=[0, 2, 1, 3])
+
+    def __combine_heads(x):
+        """
+        Transpose and then reshape the last two dimensions of inpunt tensor x
+        so that it becomes one dimension, which is reverse to __split_heads.
+        """
+        if len(x.shape) == 3: return x
+        if len(x.shape) != 4:
+            raise ValueError("Input(x) should be a 4-D Tensor.")
+
+        trans_x = layers.transpose(x, perm=[0, 2, 1, 3])
+        # The value 0 in shape attr means copying the corresponding dimension
+        # size of the input as the output dimension size.
+        return layers.reshape(
+            x=trans_x,
+            shape=[0, 0, trans_x.shape[2] * trans_x.shape[3]],
+            inplace=True)
+
+    def scaled_dot_product_attention(q, k, v, attn_bias, d_key, dropout_rate):
+        """
+        Scaled Dot-Product Attention
+        """
+        scaled_q = layers.scale(x=q, scale=d_key**-0.5)
+        product = layers.matmul(x=scaled_q, y=k, transpose_y=True)
+        if attn_bias:
+            product += attn_bias
+        weights = layers.softmax(product)
+        if dropout_rate:
+            weights = layers.dropout(
+                weights,
+                dropout_prob=dropout_rate,
+                dropout_implementation="upscale_in_train",
+                is_test=False)
+        out = layers.matmul(weights, v)
+        return out
+
+    q, k, v = __compute_qkv(queries, keys, values, n_head, d_key, d_value)
+
+    if cache is not None:  # use cache and concat time steps
+        # Since the inplace reshape in __split_heads changes the shape of k and
+        # v, which is the cache input for next time step, reshape the cache
+        # input from the previous time step first.
+        k = cache["k"] = layers.concat(
+            [layers.reshape(
+                cache["k"], shape=[0, 0, d_model]), k], axis=1)
+        v = cache["v"] = layers.concat(
+            [layers.reshape(
+                cache["v"], shape=[0, 0, d_model]), v], axis=1)
+
+    q = __split_heads(q, n_head)
+    k = __split_heads(k, n_head)
+    v = __split_heads(v, n_head)
+
+    ctx_multiheads = scaled_dot_product_attention(q, k, v, attn_bias, d_key,
+                                                  dropout_rate)
+
+    out = __combine_heads(ctx_multiheads)
+
+    # Project back to the model size.
+    proj_out = layers.fc(input=out,
+                         size=d_model,
+                         num_flatten_dims=2,
+                         param_attr=fluid.ParamAttr(
+                             name=name + '_output_fc.w_0',
+                             initializer=param_initializer),
+                         bias_attr=name + '_output_fc.b_0')
+    return proj_out
+
+
+def positionwise_feed_forward(x,
+                              d_inner_hid,
+                              d_hid,
+                              dropout_rate,
+                              hidden_act,
+                              param_initializer=None,
+                              name='ffn'):
+    """
+    Position-wise Feed-Forward Networks.
+    This module consists of two linear transformations with a ReLU activation
+    in between, which is applied to each position separately and identically.
+    """
+    hidden = layers.fc(input=x,
+                       size=d_inner_hid,
+                       num_flatten_dims=2,
+                       act=hidden_act,
+                       param_attr=fluid.ParamAttr(
+                           name=name + '_fc_0.w_0',
+                           initializer=param_initializer),
+                       bias_attr=name + '_fc_0.b_0')
+    if dropout_rate:
+        hidden = layers.dropout(
+            hidden,
+            dropout_prob=dropout_rate,
+            dropout_implementation="upscale_in_train",
+            is_test=False)
+    out = layers.fc(input=hidden,
+                    size=d_hid,
+                    num_flatten_dims=2,
+                    param_attr=fluid.ParamAttr(
+                        name=name + '_fc_1.w_0', initializer=param_initializer),
+                    bias_attr=name + '_fc_1.b_0')
+    return out
+
+
+def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.,
+                           name=''):
+    """
+    Add residual connection, layer normalization and droput to the out tensor
+    optionally according to the value of process_cmd.
+    This will be used before or after multi-head attention and position-wise
+    feed-forward networks.
+    """
+    for cmd in process_cmd:
+        if cmd == "a":  # add residual connection
+            out = out + prev_out if prev_out else out
+        elif cmd == "n":  # add layer normalization
+            out_dtype = out.dtype
+            if out_dtype == fluid.core.VarDesc.VarType.FP16:
+                out = layers.cast(x=out, dtype="float32")
+            out = layers.layer_norm(
+                out,
+                begin_norm_axis=len(out.shape) - 1,
+                param_attr=fluid.ParamAttr(
+                    name=name + '_layer_norm_scale',
+                    initializer=fluid.initializer.Constant(1.)),
+                bias_attr=fluid.ParamAttr(
+                    name=name + '_layer_norm_bias',
+                    initializer=fluid.initializer.Constant(0.)))
+            if out_dtype == fluid.core.VarDesc.VarType.FP16:
+                out = layers.cast(x=out, dtype="float16")
+        elif cmd == "d":  # add dropout
+            if dropout_rate:
+                out = layers.dropout(
+                    out,
+                    dropout_prob=dropout_rate,
+                    dropout_implementation="upscale_in_train",
+                    is_test=False)
+    return out
+
+
+pre_process_layer = partial(pre_post_process_layer, None)
+post_process_layer = pre_post_process_layer
+
+
+def encoder_layer(enc_input,
+                  attn_bias,
+                  n_head,
+                  d_key,
+                  d_value,
+                  d_model,
+                  d_inner_hid,
+                  prepostprocess_dropout,
+                  attention_dropout,
+                  relu_dropout,
+                  hidden_act,
+                  preprocess_cmd="n",
+                  postprocess_cmd="da",
+                  param_initializer=None,
+                  name=''):
+    """The encoder layers that can be stacked to form a deep encoder.
+    This module consits of a multi-head (self) attention followed by
+    position-wise feed-forward networks and both the two components companied
+    with the post_process_layer to add residual connection, layer normalization
+    and droput.
+    """
+    attn_output = multi_head_attention(
+        pre_process_layer(
+            enc_input,
+            preprocess_cmd,
+            prepostprocess_dropout,
+            name=name + '_pre_att'),
+        None,
+        None,
+        attn_bias,
+        d_key,
+        d_value,
+        d_model,
+        n_head,
+        attention_dropout,
+        param_initializer=param_initializer,
+        name=name + '_multi_head_att')
+    attn_output = post_process_layer(
+        enc_input,
+        attn_output,
+        postprocess_cmd,
+        prepostprocess_dropout,
+        name=name + '_post_att')
+    ffd_output = positionwise_feed_forward(
+        pre_process_layer(
+            attn_output,
+            preprocess_cmd,
+            prepostprocess_dropout,
+            name=name + '_pre_ffn'),
+        d_inner_hid,
+        d_model,
+        relu_dropout,
+        hidden_act,
+        param_initializer=param_initializer,
+        name=name + '_ffn')
+    return post_process_layer(
+        attn_output,
+        ffd_output,
+        postprocess_cmd,
+        prepostprocess_dropout,
+        name=name + '_post_ffn')
+
+
+def encoder(enc_input,
+            attn_bias,
+            n_layer,
+            n_head,
+            d_key,
+            d_value,
+            d_model,
+            d_inner_hid,
+            prepostprocess_dropout,
+            attention_dropout,
+            relu_dropout,
+            hidden_act,
+            preprocess_cmd="n",
+            postprocess_cmd="da",
+            param_initializer=None,
+            name=''):
+    """
+    The encoder is composed of a stack of identical layers returned by calling
+    encoder_layer.
+    """
+    for i in range(n_layer):
+        enc_output = encoder_layer(
+            enc_input,
+            attn_bias,
+            n_head,
+            d_key,
+            d_value,
+            d_model,
+            d_inner_hid,
+            prepostprocess_dropout,
+            attention_dropout,
+            relu_dropout,
+            hidden_act,
+            preprocess_cmd,
+            postprocess_cmd,
+            param_initializer=param_initializer,
+            name=name + '_layer_' + str(i))
+        enc_input = enc_output
+    enc_output = pre_process_layer(
+        enc_output, preprocess_cmd, prepostprocess_dropout, name="post_encoder")
+
+    return enc_output

BERT/optimization.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Optimization and learning rate scheduling."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import paddle.fluid as fluid
+from utils.fp16 import create_master_params_grads, master_param_to_train_param
+
+
+def linear_warmup_decay(learning_rate, warmup_steps, num_train_steps):
+    """ Applies linear warmup of learning rate from 0 and decay to 0."""
+    with fluid.default_main_program()._lr_schedule_guard():
+        lr = fluid.layers.tensor.create_global_var(
+            shape=[1],
+            value=0.0,
+            dtype='float32',
+            persistable=True,
+            name="scheduled_learning_rate")
+
+        global_step = fluid.layers.learning_rate_scheduler._decay_step_counter()
+
+        with fluid.layers.control_flow.Switch() as switch:
+            with switch.case(global_step < warmup_steps):
+                warmup_lr = learning_rate * (global_step / warmup_steps)
+                fluid.layers.tensor.assign(warmup_lr, lr)
+            with switch.default():
+                decayed_lr = fluid.layers.learning_rate_scheduler.polynomial_decay(
+                    learning_rate=learning_rate,
+                    decay_steps=num_train_steps,
+                    end_learning_rate=0.0,
+                    power=1.0,
+                    cycle=False)
+                fluid.layers.tensor.assign(decayed_lr, lr)
+
+        return lr
+
+
+def optimization(loss,
+                 warmup_steps,
+                 num_train_steps,
+                 learning_rate,
+                 train_program,
+                 startup_prog,
+                 weight_decay,
+                 scheduler='linear_warmup_decay',
+                 use_fp16=False,
+                 loss_scaling=1.0):
+    if warmup_steps > 0:
+        if scheduler == 'noam_decay':
+            scheduled_lr = fluid.layers.learning_rate_scheduler\
+             .noam_decay(1/(warmup_steps *(learning_rate ** 2)),
+                         warmup_steps)
+        elif scheduler == 'linear_warmup_decay':
+            scheduled_lr = linear_warmup_decay(learning_rate, warmup_steps,
+                                               num_train_steps)
+        else:
+            raise ValueError("Unkown learning rate scheduler, should be "
+                             "'noam_decay' or 'linear_warmup_decay'")
+        optimizer = fluid.optimizer.Adam(learning_rate=scheduled_lr)
+    else:
+        optimizer = fluid.optimizer.Adam(learning_rate=learning_rate)
+        scheduled_lr = learning_rate
+
+    clip_norm_thres = 1.0
+    # When using mixed precision training, scale the gradient clip threshold
+    # by loss_scaling
+    if use_fp16 and loss_scaling > 1.0:
+        clip_norm_thres *= loss_scaling
+    fluid.clip.set_gradient_clip(
+        clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=clip_norm_thres))
+
+    def exclude_from_weight_decay(name):
+        if name.find("layer_norm") > -1:
+            return True
+        bias_suffix = ["_bias", "_b", ".b_0"]
+        for suffix in bias_suffix:
+            if name.endswith(suffix):
+                return True
+        return False
+
+    param_list = dict()
+
+    if use_fp16:
+        param_grads = optimizer.backward(loss)
+        master_param_grads = create_master_params_grads(
+            param_grads, train_program, startup_prog, loss_scaling)
+
+        for param, _ in master_param_grads:
+            param_list[param.name] = param * 1.0
+            param_list[param.name].stop_gradient = True
+
+        optimizer.apply_gradients(master_param_grads)
+
+        if weight_decay > 0:
+            for param, grad in master_param_grads:
+                if exclude_from_weight_decay(param.name.rstrip(".master")):
+                    continue
+                with param.block.program._optimized_guard(
+                    [param, grad]), fluid.framework.name_scope("weight_decay"):
+                    updated_param = param - param_list[
+                        param.name] * weight_decay * scheduled_lr
+                    fluid.layers.assign(output=param, input=updated_param)
+
+        master_param_to_train_param(master_param_grads, param_grads,
+                                    train_program)
+
+    else:
+        for param in train_program.global_block().all_parameters():
+            param_list[param.name] = param * 1.0
+            param_list[param.name].stop_gradient = True
+
+        _, param_grads = optimizer.minimize(loss)
+
+        if weight_decay > 0:
+            for param, grad in param_grads:
+                if exclude_from_weight_decay(param.name):
+                    continue
+                with param.block.program._optimized_guard(
+                    [param, grad]), fluid.framework.name_scope("weight_decay"):
+                    updated_param = param - param_list[
+                        param.name] * weight_decay * scheduled_lr
+                    fluid.layers.assign(output=param, input=updated_param)
+
+    return scheduled_lr

BERT/predict_classifier.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Load classifier's checkpoint to do prediction or save inference model."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import time
+import argparse
+import numpy as np
+import paddle.fluid as fluid
+
+import reader.cls as reader
+from model.bert import BertConfig
+from model.classifier import create_model
+
+from utils.args import ArgumentGroup, print_arguments
+from utils.init import init_pretraining_params
+
+# yapf: disable
+parser = argparse.ArgumentParser(__doc__)
+model_g = ArgumentGroup(parser, "model", "options to init, resume and save model.")
+model_g.add_arg("bert_config_path",             str,  None,  "Path to the json file for bert model config.")
+model_g.add_arg("init_checkpoint",              str,  None,  "Init checkpoint to resume training from.")
+model_g.add_arg("save_inference_model_path",    str,  None,  "If set, save the inference model to this path.")
+model_g.add_arg("use_fp16",                     bool, False, "Whether to resume parameters from fp16 checkpoint.")
+
+data_g = ArgumentGroup(parser, "data", "Data paths, vocab paths and data processing options.")
+data_g.add_arg("data_dir",      str,  None,  "Directory to test data.")
+data_g.add_arg("vocab_path",    str,  None,  "Vocabulary path.")
+data_g.add_arg("max_seq_len",   int,  512,   "Number of words of the longest seqence.")
+data_g.add_arg("batch_size",    int,  32,    "Total examples' number in batch for training. see also --in_tokens.")
+data_g.add_arg("in_tokens",     bool, False,
+              "If set, the batch size will be the maximum number of tokens in one batch. "
+              "Otherwise, it will be the maximum number of examples in one batch.")
+data_g.add_arg("do_lower_case", bool, True,
+               "Whether to lower case the input text. Should be True for uncased models and False for cased models.")
+
+run_type_g = ArgumentGroup(parser, "run_type", "running type options.")
+run_type_g.add_arg("use_cuda",          bool,   True,  "If set, use GPU for training.")
+run_type_g.add_arg("use_fast_executor", bool,   False, "If set, use fast parallel executor (in experiment).")
+run_type_g.add_arg("task_name",         str,    None,
+                   "The name of task to perform fine-tuning, should be in {'xnli', 'mnli', 'cola', 'mrpc'}.")
+run_type_g.add_arg("do_prediction",     bool,   True,  "Whether to do prediction on test set.")
+
+args = parser.parse_args()
+# yapf: enable.
+
+def main(args):
+    bert_config = BertConfig(args.bert_config_path)
+    bert_config.print_config()
+
+    task_name = args.task_name.lower()
+    processors = {
+        'xnli': reader.XnliProcessor,
+        'cola': reader.ColaProcessor,
+        'mrpc': reader.MrpcProcessor,
+        'mnli': reader.MnliProcessor,
+    }
+
+    processor = processors[task_name](data_dir=args.data_dir,
+                                      vocab_path=args.vocab_path,
+                                      max_seq_len=args.max_seq_len,
+                                      do_lower_case=args.do_lower_case,
+                                      in_tokens=False)
+    num_labels = len(processor.get_labels())
+
+    predict_prog = fluid.Program()
+    predict_startup = fluid.Program()
+    with fluid.program_guard(predict_prog, predict_startup):
+        with fluid.unique_name.guard():
+            predict_pyreader, probs, feed_target_names = create_model(
+                args,
+                pyreader_name='predict_reader',
+                bert_config=bert_config,
+                num_labels=num_labels,
+                is_prediction=True)
+
+    predict_prog = predict_prog.clone(for_test=True)
+
+    if args.use_cuda:
+        place = fluid.CUDAPlace(0)
+        dev_count = fluid.core.get_cuda_device_count()
+    else:
+        place = fluid.CPUPlace()
+        dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
+
+    place = fluid.CUDAPlace(0) if args.use_cuda == True else fluid.CPUPlace()
+    exe = fluid.Executor(place)
+    exe.run(predict_startup)
+
+    if args.init_checkpoint:
+        init_pretraining_params(exe, args.init_checkpoint, predict_prog)
+    else:
+        raise ValueError("args 'init_checkpoint' should be set for prediction!")
+
+    # Due to the design that ParallelExecutor would drop small batches (mostly the last batch)
+    # So using ParallelExecutor may left some data unpredicted
+    # if prediction of each and every example is needed, use Executor instead
+    predict_exe = fluid.ParallelExecutor(
+        use_cuda=args.use_cuda, main_program=predict_prog)
+
+    predict_pyreader.decorate_tensor_provider(
+        processor.data_generator(
+            batch_size=args.batch_size, phase='test', epoch=1))
+
+    predict_pyreader.start()
+    all_results = []
+    time_begin = time.time()
+    while True:
+        try:
+            results = predict_exe.run(fetch_list=[probs.name])
+            all_results.extend(results[0])
+        except fluid.core.EOFException:
+            predict_pyreader.reset()
+            break
+    time_end = time.time()
+
+    np.set_printoptions(precision=4, suppress=True)
+    print("-------------- prediction results --------------")
+    print("example_id\t" + '  '.join(processor.get_labels()))
+    for index, result in enumerate(all_results):
+        print(str(index) + '\t{}'.format(result))
+
+    if args.save_inference_model_path:
+        _, ckpt_dir = os.path.split(args.init_checkpoint.rstrip('/'))
+        dir_name = ckpt_dir + '_inference_model'
+        model_path = os.path.join(args.save_inference_model_path, dir_name)
+        print("save inference model to %s" % model_path)
+        fluid.io.save_inference_model(
+            model_path,
+            feed_target_names, [probs],
+            exe,
+            main_program=predict_prog)
+
+
+if __name__ == '__main__':
+    print_arguments(args)
+    main(args)

BERT/reader/cls.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import types
+import csv
+import numpy as np
+import tokenization
+from batching import prepare_batch_data
+
+
+class DataProcessor(object):
+    """Base class for data converters for sequence classification data sets."""
+
+    def __init__(self,
+                 data_dir,
+                 vocab_path,
+                 max_seq_len,
+                 do_lower_case,
+                 in_tokens,
+                 random_seed=None):
+        self.data_dir = data_dir
+        self.max_seq_len = max_seq_len
+        self.tokenizer = tokenization.FullTokenizer(
+            vocab_file=vocab_path, do_lower_case=do_lower_case)
+        self.vocab = self.tokenizer.vocab
+        self.in_tokens = in_tokens
+
+        np.random.seed(random_seed)
+
+        self.current_train_example = -1
+        self.num_examples = {'train': -1, 'dev': -1, 'test': -1}
+        self.current_train_epoch = -1
+
+    def get_train_examples(self, data_dir):
+        """Gets a collection of `InputExample`s for the train set."""
+        raise NotImplementedError()
+
+    def get_dev_examples(self, data_dir):
+        """Gets a collection of `InputExample`s for the dev set."""
+        raise NotImplementedError()
+
+    def get_test_examples(self, data_dir):
+        """Gets a collection of `InputExample`s for prediction."""
+        raise NotImplementedError()
+
+    def get_labels(self):
+        """Gets the list of labels for this data set."""
+        raise NotImplementedError()
+
+    def convert_example(self, index, example, labels, max_seq_len, tokenizer):
+        """Converts a single `InputExample` into a single `InputFeatures`."""
+        feature = convert_single_example(index, example, labels, max_seq_len,
+                                         tokenizer)
+        return feature
+
+    def generate_instance(self, feature):
+        """
+        generate instance with given feature
+
+        Args:
+            feature: InputFeatures(object). A single set of features of data.
+        """
+        input_pos = list(range(len(feature.input_ids)))
+        return [
+            feature.input_ids, feature.segment_ids, input_pos, feature.label_id
+        ]
+
+    def generate_batch_data(self,
+                            batch_data,
+                            total_token_num,
+                            voc_size=-1,
+                            mask_id=-1,
+                            return_attn_bias=True,
+                            return_max_len=False,
+                            return_num_token=False):
+        return prepare_batch_data(
+            batch_data,
+            total_token_num,
+            voc_size=-1,
+            pad_id=self.vocab["[PAD]"],
+            cls_id=self.vocab["[CLS]"],
+            sep_id=self.vocab["[SEP]"],
+            mask_id=-1,
+            return_attn_bias=True,
+            return_max_len=False,
+            return_num_token=False)
+
+    @classmethod
+    def _read_tsv(cls, input_file, quotechar=None):
+        """Reads a tab separated value file."""
+        with open(input_file, "r") as f:
+            reader = csv.reader(f, delimiter="\t", quotechar=quotechar)
+            lines = []
+            for line in reader:
+                lines.append(line)
+            return lines
+
+    def get_num_examples(self, phase):
+        """Get number of examples for train, dev or test."""
+        if phase not in ['train', 'dev', 'test']:
+            raise ValueError(
+                "Unknown phase, which should be in ['train', 'dev', 'test'].")
+        return self.num_examples[phase]
+
+    def get_train_progress(self):
+        """Gets progress for training phase."""
+        return self.current_train_example, self.current_train_epoch
+
+    def data_generator(self, batch_size, phase='train', epoch=1, shuffle=True):
+        """
+        Generate data for train, dev or test.
+    
+        Args:
+          batch_size: int. The batch size of generated data.
+          phase: string. The phase for which to generate data.
+          epoch: int. Total epoches to generate data.
+          shuffle: bool. Whether to shuffle examples.
+        """
+        if phase == 'train':
+            examples = self.get_train_examples(self.data_dir)
+            self.num_examples['train'] = len(examples)
+        elif phase == 'dev':
+            examples = self.get_dev_examples(self.data_dir)
+            self.num_examples['dev'] = len(examples)
+        elif phase == 'test':
+            examples = self.get_test_examples(self.data_dir)
+            self.num_examples['test'] = len(examples)
+        else:
+            raise ValueError(
+                "Unknown phase, which should be in ['train', 'dev', 'test'].")
+
+        def instance_reader():
+            for epoch_index in range(epoch):
+                if shuffle:
+                    np.random.shuffle(examples)
+                if phase == 'train':
+                    self.current_train_epoch = epoch_index
+                for (index, example) in enumerate(examples):
+                    if phase == 'train':
+                        self.current_train_example = index + 1
+                    feature = self.convert_example(
+                        index, example,
+                        self.get_labels(), self.max_seq_len, self.tokenizer)
+
+                    instance = self.generate_instance(feature)
+                    yield instance
+
+        def batch_reader(reader, batch_size, in_tokens):
+            batch, total_token_num, max_len = [], 0, 0
+            for instance in reader():
+                token_ids, sent_ids, pos_ids, label = instance[:4]
+                max_len = max(max_len, len(token_ids))
+                if in_tokens:
+                    to_append = (len(batch) + 1) * max_len <= batch_size
+                else:
+                    to_append = len(batch) < batch_size
+                if to_append:
+                    batch.append(instance)
+                    total_token_num += len(token_ids)
+                else:
+                    yield batch, total_token_num
+                    batch, total_token_num, max_len = [instance], len(
+                        token_ids), len(token_ids)
+
+            if len(batch) > 0:
+                yield batch, total_token_num
+
+        def wrapper():
+            for batch_data, total_token_num in batch_reader(
+                    instance_reader, batch_size, self.in_tokens):
+                batch_data = self.generate_batch_data(
+                    batch_data,
+                    total_token_num,
+                    voc_size=-1,
+                    mask_id=-1,
+                    return_attn_bias=True,
+                    return_max_len=False,
+                    return_num_token=False)
+                yield batch_data
+
+        return wrapper
+
+
+class InputExample(object):
+    """A single training/test example for simple sequence classification."""
+
+    def __init__(self, guid, text_a, text_b=None, label=None):
+        """Constructs a InputExample.
+
+    Args:
+      guid: Unique id for the example.
+      text_a: string. The untokenized text of the first sequence. For single
+        sequence tasks, only this sequence must be specified.
+      text_b: (Optional) string. The untokenized text of the second sequence.
+        Only must be specified for sequence pair tasks.
+      label: (Optional) string. The label of the example. This should be
+        specified for train and dev examples, but not for test examples.
+    """
+        self.guid = guid
+        self.text_a = text_a
+        self.text_b = text_b
+        self.label = label
+
+
+def _truncate_seq_pair(tokens_a, tokens_b, max_length):
+    """Truncates a sequence pair in place to the maximum length."""
+
+    # This is a simple heuristic which will always truncate the longer sequence
+    # one token at a time. This makes more sense than truncating an equal percent
+    # of tokens from each, since if one sequence is very short then each token
+    # that's truncated likely contains more information than a longer sequence.
+    while True:
+        total_length = len(tokens_a) + len(tokens_b)
+        if total_length <= max_length:
+            break
+        if len(tokens_a) > len(tokens_b):
+            tokens_a.pop()
+        else:
+            tokens_b.pop()
+
+
+class InputFeatures(object):
+    """A single set of features of data."""
+
+    def __init__(self, input_ids, input_mask, segment_ids, label_id):
+        self.input_ids = input_ids
+        self.input_mask = input_mask
+        self.segment_ids = segment_ids
+        self.label_id = label_id
+
+
+class XnliProcessor(DataProcessor):
+    """Processor for the XNLI data set."""
+
+    def get_train_examples(self, data_dir):
+        """See base class."""
+        self.language = "zh"
+        lines = self._read_tsv(
+            os.path.join(data_dir, "multinli", "multinli.train.%s.tsv" %
+                         self.language))
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            guid = "train-%d" % (i)
+            text_a = tokenization.convert_to_unicode(line[0])
+            text_b = tokenization.convert_to_unicode(line[1])
+            label = tokenization.convert_to_unicode(line[2])
+            if label == tokenization.convert_to_unicode("contradictory"):
+                label = tokenization.convert_to_unicode("contradiction")
+            examples.append(
+                InputExample(
+                    guid=guid, text_a=text_a, text_b=text_b, label=label))
+        return examples
+
+    def get_dev_examples(self, data_dir):
+        """See base class."""
+        self.language = "zh"
+        lines = self._read_tsv(os.path.join(data_dir, "xnli.dev.tsv"))
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            guid = "dev-%d" % (i)
+            language = tokenization.convert_to_unicode(line[0])
+            if language != tokenization.convert_to_unicode(self.language):
+                continue
+            text_a = tokenization.convert_to_unicode(line[6])
+            text_b = tokenization.convert_to_unicode(line[7])
+            label = tokenization.convert_to_unicode(line[1])
+            examples.append(
+                InputExample(
+                    guid=guid, text_a=text_a, text_b=text_b, label=label))
+        return examples
+
+    def get_test_examples(self, data_dir):
+        """See base class."""
+        self.language = "zh"
+        lines = self._read_tsv(os.path.join(data_dir, "xnli.test.tsv"))
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            guid = "test-%d" % (i)
+            language = tokenization.convert_to_unicode(line[0])
+            if language != tokenization.convert_to_unicode(self.language):
+                continue
+            text_a = tokenization.convert_to_unicode(line[6])
+            text_b = tokenization.convert_to_unicode(line[7])
+            label = tokenization.convert_to_unicode(line[1])
+            examples.append(
+                InputExample(
+                    guid=guid, text_a=text_a, text_b=text_b, label=label))
+        return examples
+
+    def get_labels(self):
+        """See base class."""
+        return ["contradiction", "entailment", "neutral"]
+
+
+class MnliProcessor(DataProcessor):
+    """Processor for the MultiNLI data set (GLUE version)."""
+
+    def get_train_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")
+
+    def get_dev_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")),
+            "dev_matched")
+
+    def get_test_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "test_matched.tsv")), "test")
+
+    def get_labels(self):
+        """See base class."""
+        return ["contradiction", "entailment", "neutral"]
+
+    def _create_examples(self, lines, set_type):
+        """Creates examples for the training and dev sets."""
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            guid = "%s-%s" % (set_type,
+                              tokenization.convert_to_unicode(line[0]))
+            text_a = tokenization.convert_to_unicode(line[8])
+            text_b = tokenization.convert_to_unicode(line[9])
+            if set_type == "test":
+                label = "contradiction"
+            else:
+                label = tokenization.convert_to_unicode(line[-1])
+            examples.append(
+                InputExample(
+                    guid=guid, text_a=text_a, text_b=text_b, label=label))
+        return examples
+
+
+class MrpcProcessor(DataProcessor):
+    """Processor for the MRPC data set (GLUE version)."""
+
+    def get_train_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")
+
+    def get_dev_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")
+
+    def get_test_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")
+
+    def get_labels(self):
+        """See base class."""
+        return ["0", "1"]
+
+    def _create_examples(self, lines, set_type):
+        """Creates examples for the training and dev sets."""
+        examples = []
+        for (i, line) in enumerate(lines):
+            if i == 0:
+                continue
+            guid = "%s-%s" % (set_type, i)
+            text_a = tokenization.convert_to_unicode(line[3])
+            text_b = tokenization.convert_to_unicode(line[4])
+            if set_type == "test":
+                label = "0"
+            else:
+                label = tokenization.convert_to_unicode(line[0])
+            examples.append(
+                InputExample(
+                    guid=guid, text_a=text_a, text_b=text_b, label=label))
+        return examples
+
+
+class ColaProcessor(DataProcessor):
+    """Processor for the CoLA data set (GLUE version)."""
+
+    def get_train_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")
+
+    def get_dev_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")
+
+    def get_test_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")
+
+    def get_labels(self):
+        """See base class."""
+        return ["0", "1"]
+
+    def _create_examples(self, lines, set_type):
+        """Creates examples for the training and dev sets."""
+        examples = []
+        for (i, line) in enumerate(lines):
+            # Only the test set has a header
+            if set_type == "test" and i == 0:
+                continue
+            guid = "%s-%s" % (set_type, i)
+            if set_type == "test":
+                text_a = tokenization.convert_to_unicode(line[1])
+                label = "0"
+            else:
+                text_a = tokenization.convert_to_unicode(line[3])
+                label = tokenization.convert_to_unicode(line[1])
+            examples.append(
+                InputExample(
+                    guid=guid, text_a=text_a, text_b=None, label=label))
+        return examples
+
+
+def convert_single_example_to_unicode(guid, single_example):
+    text_a = tokenization.convert_to_unicode(single_example[0])
+    text_b = tokenization.convert_to_unicode(single_example[1])
+    label = tokenization.convert_to_unicode(single_example[2])
+    return InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)
+
+
+def convert_single_example(ex_index, example, label_list, max_seq_length,
+                           tokenizer):
+    """Converts a single `InputExample` into a single `InputFeatures`."""
+    label_map = {}
+    for (i, label) in enumerate(label_list):
+        label_map[label] = i
+
+    tokens_a = tokenizer.tokenize(example.text_a)
+    tokens_b = None
+    if example.text_b:
+        tokens_b = tokenizer.tokenize(example.text_b)
+
+    if tokens_b:
+        # Modifies `tokens_a` and `tokens_b` in place so that the total
+        # length is less than the specified length.
+        # Account for [CLS], [SEP], [SEP] with "- 3"
+        _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
+    else:
+        # Account for [CLS] and [SEP] with "- 2"
+        if len(tokens_a) > max_seq_length - 2:
+            tokens_a = tokens_a[0:(max_seq_length - 2)]
+
+    # The convention in BERT is:
+    # (a) For sequence pairs:
+    #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
+    #  type_ids: 0     0  0    0    0     0       0 0     1  1  1  1   1 1
+    # (b) For single sequences:
+    #  tokens:   [CLS] the dog is hairy . [SEP]
+    #  type_ids: 0     0   0   0  0     0 0
+    #
+    # Where "type_ids" are used to indicate whether this is the first
+    # sequence or the second sequence. The embedding vectors for `type=0` and
+    # `type=1` were learned during pre-training and are added to the wordpiece
+    # embedding vector (and position vector). This is not *strictly* necessary
+    # since the [SEP] token unambiguously separates the sequences, but it makes
+    # it easier for the model to learn the concept of sequences.
+    #
+    # For classification tasks, the first vector (corresponding to [CLS]) is
+    # used as as the "sentence vector". Note that this only makes sense because
+    # the entire model is fine-tuned.
+    tokens = []
+    segment_ids = []
+    tokens.append("[CLS]")
+    segment_ids.append(0)
+    for token in tokens_a:
+        tokens.append(token)
+        segment_ids.append(0)
+    tokens.append("[SEP]")
+    segment_ids.append(0)
+
+    if tokens_b:
+        for token in tokens_b:
+            tokens.append(token)
+            segment_ids.append(1)
+        tokens.append("[SEP]")
+        segment_ids.append(1)
+
+    input_ids = tokenizer.convert_tokens_to_ids(tokens)
+
+    # The mask has 1 for real tokens and 0 for padding tokens. Only real
+    # tokens are attended to.
+    input_mask = [1] * len(input_ids)
+
+    label_id = label_map[example.label]
+
+    feature = InputFeatures(
+        input_ids=input_ids,
+        input_mask=input_mask,
+        segment_ids=segment_ids,
+        label_id=label_id)
+    return feature
+
+
+def convert_examples_to_features(examples, label_list, max_seq_length,
+                                 tokenizer):
+    """Convert a set of `InputExample`s to a list of `InputFeatures`."""
+
+    features = []
+    for (ex_index, example) in enumerate(examples):
+        if ex_index % 10000 == 0:
+            print("Writing example %d of %d" % (ex_index, len(examples)))
+
+        feature = convert_single_example(ex_index, example, label_list,
+                                         max_seq_length, tokenizer)
+
+        features.append(feature)
+    return features
+
+
+if __name__ == '__main__':
+    pass

BERT/reader/pretraining.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+from __future__ import division
+
+import os
+import numpy as np
+import types
+import gzip
+import logging
+import re
+import six
+import collections
+import tokenization
+
+import paddle
+import paddle.fluid as fluid
+
+from batching import prepare_batch_data
+
+
+class DataReader(object):
+    def __init__(self,
+                 data_dir,
+                 vocab_path,
+                 batch_size=4096,
+                 max_seq_len=512,
+                 shuffle_files=True,
+                 epoch=100,
+                 voc_size=0,
+                 is_test=False,
+                 generate_neg_sample=False):
+
+        self.vocab = self.load_vocab(vocab_path)
+        self.data_dir = data_dir
+        self.batch_size = batch_size
+        self.shuffle_files = shuffle_files
+        self.epoch = epoch
+        self.current_epoch = 0
+        self.current_file_index = 0
+        self.total_file = 0
+        self.current_file = None
+        self.voc_size = voc_size
+        self.max_seq_len = max_seq_len
+        self.pad_id = self.vocab["[PAD]"]
+        self.cls_id = self.vocab["[CLS]"]
+        self.sep_id = self.vocab["[SEP]"]
+        self.mask_id = self.vocab["[MASK]"]
+        self.is_test = is_test
+        self.generate_neg_sample = generate_neg_sample
+        assert self.batch_size > 100, "Current batch size means total token's number, \
+                                       it should not be set to too small number."
+
+        if self.is_test:
+            self.epoch = 1
+            self.shuffle_files = False
+
+    def get_progress(self):
+        """return current progress of traning data
+        """
+        return self.current_epoch, self.current_file_index, self.total_file, self.current_file
+
+    def parse_line(self, line, max_seq_len=512):
+        """ parse one line to token_ids, sentence_ids, pos_ids, label
+        """
+        line = line.strip().split(";")
+        assert len(line) == 4, "One sample must have 4 fields!"
+        (token_ids, sent_ids, pos_ids, label) = line
+        token_ids = [int(token) for token in token_ids.split(" ")]
+        sent_ids = [int(token) for token in sent_ids.split(" ")]
+        pos_ids = [int(token) for token in pos_ids.split(" ")]
+        assert len(token_ids) == len(sent_ids) == len(
+            pos_ids
+        ), "[Must be true]len(token_ids) == len(sent_ids) == len(pos_ids)"
+        label = int(label)
+        if len(token_ids) > max_seq_len:
+            return None
+        return [token_ids, sent_ids, pos_ids, label]
+
+    def read_file(self, file):
+        assert file.endswith('.gz'), "[ERROR] %s is not a gzip file" % file
+        file_path = self.data_dir + "/" + file
+        with gzip.open(file_path, "rb") as f:
+            for line in f:
+                parsed_line = self.parse_line(
+                    line, max_seq_len=self.max_seq_len)
+                if parsed_line is None:
+                    continue
+                yield parsed_line
+
+    def convert_to_unicode(self, text):
+        """Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
+        if six.PY3:
+            if isinstance(text, str):
+                return text
+            elif isinstance(text, bytes):
+                return text.decode("utf-8", "ignore")
+            else:
+                raise ValueError("Unsupported string type: %s" % (type(text)))
+        elif six.PY2:
+            if isinstance(text, str):
+                return text.decode("utf-8", "ignore")
+            elif isinstance(text, unicode):
+                return text
+            else:
+                raise ValueError("Unsupported string type: %s" % (type(text)))
+        else:
+            raise ValueError("Not running on Python2 or Python 3?")
+
+    def load_vocab(self, vocab_file):
+        """Loads a vocabulary file into a dictionary."""
+        vocab = collections.OrderedDict()
+        fin = open(vocab_file)
+        for num, line in enumerate(fin):
+            items = self.convert_to_unicode(line.strip()).split("\t")
+            if len(items) > 2:
+                break
+            token = items[0]
+            index = items[1] if len(items) == 2 else num
+            token = token.strip()
+            vocab[token] = int(index)
+        return vocab
+
+    def random_pair_neg_samples(self, pos_samples):
+        """ randomly generate negtive samples using pos_samples
+
+            Args:
+                pos_samples: list of positive samples
+            
+            Returns:
+                neg_samples: list of negtive samples
+        """
+        np.random.shuffle(pos_samples)
+        num_sample = len(pos_samples)
+        neg_samples = []
+        miss_num = 0
+
+        for i in range(num_sample):
+            pair_index = (i + 1) % num_sample
+            origin_src_ids = pos_samples[i][0]
+            origin_sep_index = origin_src_ids.index(2)
+            pair_src_ids = pos_samples[pair_index][0]
+            pair_sep_index = pair_src_ids.index(2)
+
+            src_ids = origin_src_ids[:origin_sep_index + 1] + pair_src_ids[
+                pair_sep_index + 1:]
+            if len(src_ids) >= self.max_seq_len:
+                miss_num += 1
+                continue
+            sent_ids = [0] * len(origin_src_ids[:origin_sep_index + 1]) + [
+                1
+            ] * len(pair_src_ids[pair_sep_index + 1:])
+            pos_ids = list(range(len(src_ids)))
+            neg_sample = [src_ids, sent_ids, pos_ids, 0]
+            assert len(src_ids) == len(sent_ids) == len(
+                pos_ids
+            ), "[ERROR]len(src_id) == lne(sent_id) == len(pos_id) must be True"
+            neg_samples.append(neg_sample)
+        return neg_samples, miss_num
+
+    def mixin_negtive_samples(self, pos_sample_generator, buffer=1000):
+        """ 1. generate negtive samples by randomly group sentence_1 and sentence_2 of positive samples
+            2. combine negtive samples and positive samples
+            
+            Args:
+                pos_sample_generator: a generator producing a parsed positive sample, which is a list: [token_ids, sent_ids, pos_ids, 1]
+
+            Returns:
+                sample: one sample from shuffled positive samples and negtive samples
+        """
+        pos_samples = []
+        num_total_miss = 0
+        pos_sample_num = 0
+        try:
+            while True:
+                while len(pos_samples) < buffer:
+                    pos_sample = next(pos_sample_generator)
+                    label = pos_sample[3]
+                    assert label == 1, "positive sample's label must be 1"
+                    pos_samples.append(pos_sample)
+                    pos_sample_num += 1
+
+                neg_samples, miss_num = self.random_pair_neg_samples(
+                    pos_samples)
+                num_total_miss += miss_num
+                samples = pos_samples + neg_samples
+                pos_samples = []
+                np.random.shuffle(samples)
+                for sample in samples:
+                    yield sample
+        except StopIteration:
+            print("stopiteration: reach end of file")
+            if len(pos_samples) == 1:
+                yield pos_samples[0]
+            elif len(pos_samples) == 0:
+                yield None
+            else:
+                neg_samples, miss_num = self.random_pair_neg_samples(
+                    pos_samples)
+                num_total_miss += miss_num
+                samples = pos_samples + neg_samples
+                pos_samples = []
+                np.random.shuffle(samples)
+                for sample in samples:
+                    yield sample
+            print("miss_num:%d\tideal_total_sample_num:%d\tmiss_rate:%f" %
+                  (num_total_miss, pos_sample_num * 2,
+                   num_total_miss / (pos_sample_num * 2)))
+
+    def data_generator(self):
+        """
+        data_generator
+        """
+        files = os.listdir(self.data_dir)
+        self.total_file = len(files)
+        assert self.total_file > 0, "[Error] data_dir is empty"
+
+        def wrapper():
+            def reader():
+                for epoch in range(self.epoch):
+                    self.current_epoch = epoch + 1
+                    if self.shuffle_files:
+                        np.random.shuffle(files)
+                    for index, file in enumerate(files):
+                        self.current_file_index = index + 1
+                        self.current_file = file
+                        sample_generator = self.read_file(file)
+                        if not self.is_test and self.generate_neg_sample:
+                            sample_generator = self.mixin_negtive_samples(
+                                sample_generator)
+                        for sample in sample_generator:
+                            if sample is None:
+                                continue
+                            yield sample
+
+            def batch_reader(reader, batch_size):
+                batch, total_token_num, max_len = [], 0, 0
+                for parsed_line in reader():
+                    token_ids, sent_ids, pos_ids, label = parsed_line
+                    max_len = max(max_len, len(token_ids))
+                    if (len(batch) + 1) * max_len <= batch_size:
+                        batch.append(parsed_line)
+                        total_token_num += len(token_ids)
+                    else:
+                        yield batch, total_token_num
+                        batch, total_token_num, max_len = [parsed_line], len(
+                            token_ids), len(token_ids)
+
+                if len(batch) > 0:
+                    yield batch, total_token_num
+
+            for batch_data, total_token_num in batch_reader(reader,
+                                                            self.batch_size):
+                yield prepare_batch_data(
+                    batch_data,
+                    total_token_num,
+                    voc_size=self.voc_size,
+                    pad_id=self.pad_id,
+                    cls_id=self.cls_id,
+                    sep_id=self.sep_id,
+                    mask_id=self.mask_id,
+                    return_attn_bias=True,
+                    return_max_len=False,
+                    return_num_token=False)
+
+        return wrapper
+
+
+if __name__ == "__main__":
+    pass

BERT/run_classifier.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Finetuning on classification tasks."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import time
+import argparse
+import numpy as np
+import multiprocessing
+
+import paddle
+import paddle.fluid as fluid
+
+import reader.cls as reader
+from model.bert import BertConfig
+from model.classifier import create_model
+from optimization import optimization
+from utils.args import ArgumentGroup, print_arguments
+from utils.init import init_pretraining_params, init_checkpoint
+
+# yapf: disable
+parser = argparse.ArgumentParser(__doc__)
+model_g = ArgumentGroup(parser, "model", "model configuration and paths.")
+model_g.add_arg("bert_config_path",         str,  None,           "Path to the json file for bert model config.")
+model_g.add_arg("init_checkpoint",          str,  None,           "Init checkpoint to resume training from.")
+model_g.add_arg("init_pretraining_params",  str,  None,
+                "Init pre-training params which preforms fine-tuning from. If the "
+                 "arg 'init_checkpoint' has been set, this argument wouldn't be valid.")
+model_g.add_arg("checkpoints",              str,  "checkpoints",  "Path to save checkpoints.")
+
+train_g = ArgumentGroup(parser, "training", "training options.")
+train_g.add_arg("epoch",             int,    100,     "Number of epoches for training.")
+train_g.add_arg("learning_rate",     float,  5e-5,    "Learning rate used to train with warmup.")
+train_g.add_arg("lr_scheduler",      str,    "linear_warmup_decay",
+                "scheduler of learning rate.", choices=['linear_warmup_decay', 'noam_decay'])
+train_g.add_arg("weight_decay",      float,  0.01,    "Weight decay rate for L2 regularizer.")
+train_g.add_arg("warmup_proportion", float,  0.1,
+                "Proportion of training steps to perform linear learning rate warmup for.")
+train_g.add_arg("save_steps",        int,    10000,   "The steps interval to save checkpoints.")
+train_g.add_arg("validation_steps",  int,    1000,    "The steps interval to evaluate model performance.")
+train_g.add_arg("use_fp16",          bool,   False,   "Whether to use fp16 mixed precision training.")
+train_g.add_arg("loss_scaling",      float,  1.0,
+                "Loss scaling factor for mixed precision training, only valid when use_fp16 is enabled.")
+
+log_g = ArgumentGroup(parser,     "logging", "logging related.")
+log_g.add_arg("skip_steps",          int,    10,    "The steps interval to print loss.")
+log_g.add_arg("verbose",             bool,   False, "Whether to output verbose log.")
+
+data_g = ArgumentGroup(parser, "data", "Data paths, vocab paths and data processing options")
+data_g.add_arg("data_dir",      str,  None,  "Path to training data.")
+data_g.add_arg("vocab_path",    str,  None,  "Vocabulary path.")
+data_g.add_arg("max_seq_len",   int,  512,   "Number of words of the longest seqence.")
+data_g.add_arg("batch_size",    int,  8192,  "Total examples' number in batch for training. see also --in_tokens.")
+data_g.add_arg("in_tokens",     bool, False,
+              "If set, the batch size will be the maximum number of tokens in one batch. "
+              "Otherwise, it will be the maximum number of examples in one batch.")
+data_g.add_arg("do_lower_case", bool, True,
+               "Whether to lower case the input text. Should be True for uncased models and False for cased models.")
+data_g.add_arg("random_seed",   int,  0,  "Random seed.")
+
+run_type_g = ArgumentGroup(parser, "run_type", "running type options.")
+run_type_g.add_arg("use_cuda",                     bool,   True,  "If set, use GPU for training.")
+run_type_g.add_arg("use_fast_executor",            bool,   False, "If set, use fast parallel executor (in experiment).")
+run_type_g.add_arg("num_iteration_per_drop_scope", int,    10,    "Iteration intervals to drop scope.")
+run_type_g.add_arg("task_name",                    str,    None,
+                   "The name of task to perform fine-tuning, should be in {'xnli', 'mnli', 'cola', 'mrpc'}.")
+run_type_g.add_arg("do_train",                     bool,   True,  "Whether to perform training.")
+run_type_g.add_arg("do_val",                       bool,   True,  "Whether to perform evaluation on dev data set.")
+run_type_g.add_arg("do_test",                      bool,   True,  "Whether to perform evaluation on test data set.")
+
+args = parser.parse_args()
+# yapf: enable.
+
+
+def evaluate(exe, test_program, test_pyreader, fetch_list, eval_phase):
+    test_pyreader.start()
+    total_cost, total_acc, total_num_seqs = [], [], []
+    time_begin = time.time()
+    while True:
+        try:
+            np_loss, np_acc, np_num_seqs = exe.run(program=test_program,
+                                                   fetch_list=fetch_list)
+            total_cost.extend(np_loss * np_num_seqs)
+            total_acc.extend(np_acc * np_num_seqs)
+            total_num_seqs.extend(np_num_seqs)
+        except fluid.core.EOFException:
+            test_pyreader.reset()
+            break
+    time_end = time.time()
+    print("[%s evaluation] ave loss: %f, ave acc: %f, elapsed time: %f s" %
+          (eval_phase, np.sum(total_cost) / np.sum(total_num_seqs),
+           np.sum(total_acc) / np.sum(total_num_seqs), time_end - time_begin))
+
+
+def main(args):
+    bert_config = BertConfig(args.bert_config_path)
+    bert_config.print_config()
+
+    if args.use_cuda:
+        place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0')))
+        dev_count = fluid.core.get_cuda_device_count()
+    else:
+        place = fluid.CPUPlace()
+        dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
+    exe = fluid.Executor(place)
+
+    task_name = args.task_name.lower()
+    processors = {
+        'xnli': reader.XnliProcessor,
+        'cola': reader.ColaProcessor,
+        'mrpc': reader.MrpcProcessor,
+        'mnli': reader.MnliProcessor,
+    }
+
+    processor = processors[task_name](data_dir=args.data_dir,
+                                      vocab_path=args.vocab_path,
+                                      max_seq_len=args.max_seq_len,
+                                      do_lower_case=args.do_lower_case,
+                                      in_tokens=args.in_tokens,
+                                      random_seed=args.random_seed)
+    num_labels = len(processor.get_labels())
+
+    if not (args.do_train or args.do_val or args.do_test):
+        raise ValueError("For args `do_train`, `do_val` and `do_test`, at "
+                         "least one of them must be True.")
+
+    startup_prog = fluid.Program()
+    if args.random_seed is not None:
+        startup_prog.random_seed = args.random_seed
+
+    if args.do_train:
+        train_data_generator = processor.data_generator(
+            batch_size=args.batch_size,
+            phase='train',
+            epoch=args.epoch,
+            shuffle=True)
+
+        num_train_examples = processor.get_num_examples(phase='train')
+
+        if args.in_tokens:
+            max_train_steps = args.epoch * num_train_examples // (
+                args.batch_size // args.max_seq_len) // dev_count
+        else:
+            max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count
+
+        warmup_steps = int(max_train_steps * args.warmup_proportion)
+        print("Device count: %d" % dev_count)
+        print("Num train examples: %d" % num_train_examples)
+        print("Max train steps: %d" % max_train_steps)
+        print("Num warmup steps: %d" % warmup_steps)
+
+        train_program = fluid.Program()
+
+        with fluid.program_guard(train_program, startup_prog):
+            with fluid.unique_name.guard():
+                train_pyreader, loss, probs, accuracy, num_seqs = create_model(
+                    args,
+                    pyreader_name='train_reader',
+                    bert_config=bert_config,
+                    num_labels=num_labels)
+                scheduled_lr = optimization(
+                    loss=loss,
+                    warmup_steps=warmup_steps,
+                    num_train_steps=max_train_steps,
+                    learning_rate=args.learning_rate,
+                    train_program=train_program,
+                    startup_prog=startup_prog,
+                    weight_decay=args.weight_decay,
+                    scheduler=args.lr_scheduler,
+                    use_fp16=args.use_fp16,
+                    loss_scaling=args.loss_scaling)
+
+                fluid.memory_optimize(
+                    input_program=train_program,
+                    skip_opt_set=[
+                        loss.name, probs.name, accuracy.name, num_seqs.name
+                    ])
+
+        if args.verbose:
+            if args.in_tokens:
+                lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
+                    program=train_program,
+                    batch_size=args.batch_size // args.max_seq_len)
+            else:
+                lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
+                    program=train_program, batch_size=args.batch_size)
+            print("Theoretical memory usage in training: %.3f - %.3f %s" %
+                  (lower_mem, upper_mem, unit))
+
+    if args.do_val or args.do_test:
+        test_prog = fluid.Program()
+        with fluid.program_guard(test_prog, startup_prog):
+            with fluid.unique_name.guard():
+                test_pyreader, loss, probs, accuracy, num_seqs = create_model(
+                    args,
+                    pyreader_name='test_reader',
+                    bert_config=bert_config,
+                    num_labels=num_labels)
+
+        test_prog = test_prog.clone(for_test=True)
+
+    exe.run(startup_prog)
+
+    if args.do_train:
+        if args.init_checkpoint and args.init_pretraining_params:
+            print(
+                "WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
+                "both are set! Only arg 'init_checkpoint' is made valid.")
+        if args.init_checkpoint:
+            init_checkpoint(
+                exe,
+                args.init_checkpoint,
+                main_program=startup_prog,
+                use_fp16=args.use_fp16)
+        elif args.init_pretraining_params:
+            init_pretraining_params(
+                exe,
+                args.init_pretraining_params,
+                main_program=startup_prog,
+                use_fp16=args.use_fp16)
+    elif args.do_val or args.do_test:
+        if not args.init_checkpoint:
+            raise ValueError("args 'init_checkpoint' should be set if"
+                             "only doing validation or testing!")
+        init_checkpoint(
+            exe,
+            args.init_checkpoint,
+            main_program=startup_prog,
+            use_fp16=args.use_fp16)
+
+    if args.do_train:
+        exec_strategy = fluid.ExecutionStrategy()
+        if args.use_fast_executor:
+            exec_strategy.use_experimental_executor = True
+        exec_strategy.num_threads = dev_count
+        exec_strategy.num_iteration_per_drop_scope = args.num_iteration_per_drop_scope
+
+        train_exe = fluid.ParallelExecutor(
+            use_cuda=args.use_cuda,
+            loss_name=loss.name,
+            exec_strategy=exec_strategy,
+            main_program=train_program)
+
+        train_pyreader.decorate_tensor_provider(train_data_generator)
+    else:
+        train_exe = None
+
+    if args.do_val or args.do_test:
+        test_exe = fluid.ParallelExecutor(
+            use_cuda=args.use_cuda,
+            main_program=test_prog,
+            share_vars_from=train_exe)
+
+    if args.do_train:
+        train_pyreader.start()
+        steps = 0
+        total_cost, total_acc, total_num_seqs = [], [], []
+        time_begin = time.time()
+        while True:
+            try:
+                steps += 1
+                if steps % args.skip_steps == 0:
+                    if warmup_steps <= 0:
+                        fetch_list = [loss.name, accuracy.name, num_seqs.name]
+                    else:
+                        fetch_list = [
+                            loss.name, accuracy.name, scheduled_lr.name,
+                            num_seqs.name
+                        ]
+                else:
+                    fetch_list = []
+
+                outputs = train_exe.run(fetch_list=fetch_list)
+
+                if steps % args.skip_steps == 0:
+                    if warmup_steps <= 0:
+                        np_loss, np_acc, np_num_seqs = outputs
+                    else:
+                        np_loss, np_acc, np_lr, np_num_seqs = outputs
+
+                    total_cost.extend(np_loss * np_num_seqs)
+                    total_acc.extend(np_acc * np_num_seqs)
+                    total_num_seqs.extend(np_num_seqs)
+
+                    if args.verbose:
+                        verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
+                        )
+                        verbose += "learning rate: %f" % (
+                            np_lr[0]
+                            if warmup_steps > 0 else args.learning_rate)
+                        print(verbose)
+
+                    current_example, current_epoch = processor.get_train_progress(
+                    )
+                    time_end = time.time()
+                    used_time = time_end - time_begin
+                    print("epoch: %d, progress: %d/%d, step: %d, ave loss: %f, "
+                          "ave acc: %f, speed: %f steps/s" %
+                          (current_epoch, current_example, num_train_examples,
+                           steps, np.sum(total_cost) / np.sum(total_num_seqs),
+                           np.sum(total_acc) / np.sum(total_num_seqs),
+                           args.skip_steps / used_time))
+                    total_cost, total_acc, total_num_seqs = [], [], []
+                    time_begin = time.time()
+
+                if steps % args.save_steps == 0:
+                    save_path = os.path.join(args.checkpoints,
+                                             "step_" + str(steps))
+                    fluid.io.save_persistables(exe, save_path, train_program)
+
+                if steps % args.validation_steps == 0:
+                    # evaluate dev set
+                    if args.do_val:
+                        test_pyreader.decorate_tensor_provider(
+                            processor.data_generator(
+                                batch_size=args.batch_size,
+                                phase='dev',
+                                epoch=1,
+                                shuffle=False))
+                        evaluate(exe, test_prog, test_pyreader,
+                                 [loss.name, accuracy.name, num_seqs.name],
+                                 "dev")
+                    # evaluate test set
+                    if args.do_test:
+                        test_pyreader.decorate_tensor_provider(
+                            processor.data_generator(
+                                batch_size=args.batch_size,
+                                phase='test',
+                                epoch=1,
+                                shuffle=False))
+                        evaluate(exe, test_prog, test_pyreader,
+                                 [loss.name, accuracy.name, num_seqs.name],
+                                 "test")
+            except fluid.core.EOFException:
+                save_path = os.path.join(args.checkpoints, "step_" + str(steps))
+                fluid.io.save_persistables(exe, save_path, train_program)
+                train_pyreader.reset()
+                break
+
+    # final eval on dev set
+    if args.do_val:
+        test_pyreader.decorate_tensor_provider(
+            processor.data_generator(
+                batch_size=args.batch_size, phase='dev', epoch=1,
+                shuffle=False))
+        print("Final validation result:")
+        evaluate(exe, test_prog, test_pyreader,
+                 [loss.name, accuracy.name, num_seqs.name], "dev")
+
+    # final eval on test set
+    if args.do_test:
+        test_pyreader.decorate_tensor_provider(
+            processor.data_generator(
+                batch_size=args.batch_size,
+                phase='test',
+                epoch=1,
+                shuffle=False))
+        print("Final test result:")
+        evaluate(exe, test_prog, test_pyreader,
+                 [loss.name, accuracy.name, num_seqs.name], "test")
+
+
+if __name__ == '__main__':
+    print_arguments(args)
+    main(args)

BERT/run_squad.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Finetuning on SQuAD."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import time
+import argparse
+import numpy as np
+import collections
+
+import paddle
+import paddle.fluid as fluid
+
+from reader.squad import DataProcessor, write_predictions
+from model.bert import BertConfig, BertModel
+from utils.args import ArgumentGroup, print_arguments
+from optimization import optimization
+from utils.init import init_pretraining_params, init_checkpoint
+
+# yapf: disable
+parser = argparse.ArgumentParser(__doc__)
+model_g = ArgumentGroup(parser, "model", "model configuration and paths.")
+model_g.add_arg("bert_config_path",         str,  None,           "Path to the json file for bert model config.")
+model_g.add_arg("init_checkpoint",          str,  None,           "Init checkpoint to resume training from.")
+model_g.add_arg("init_pretraining_params",  str,  None,
+                "Init pre-training params which preforms fine-tuning from. If the "
+                 "arg 'init_checkpoint' has been set, this argument wouldn't be valid.")
+model_g.add_arg("checkpoints",              str,  "checkpoints",  "Path to save checkpoints.")
+
+train_g = ArgumentGroup(parser, "training", "training options.")
+train_g.add_arg("epoch",             int,    100,     "Number of epoches for training.")
+train_g.add_arg("learning_rate",     float,  5e-5,    "Learning rate used to train with warmup.")
+train_g.add_arg("lr_scheduler",      str,    "linear_warmup_decay",
+                "scheduler of learning rate.", choices=['linear_warmup_decay', 'noam_decay'])
+train_g.add_arg("weight_decay",      float,  0.01,    "Weight decay rate for L2 regularizer.")
+train_g.add_arg("warmup_proportion", float,  0.1,
+                "Proportion of training steps to perform linear learning rate warmup for.")
+train_g.add_arg("save_steps",        int,    10000,   "The steps interval to save checkpoints.")
+train_g.add_arg("validation_steps",  int,    1000,    "The steps interval to evaluate model performance.")
+train_g.add_arg("use_fp16",          bool,   False,   "Whether to use fp16 mixed precision training.")
+train_g.add_arg("loss_scaling",      float,  1.0,
+                "Loss scaling factor for mixed precision training, only valid when use_fp16 is enabled.")
+
+log_g = ArgumentGroup(parser, "logging", "logging related.")
+log_g.add_arg("skip_steps",          int,    10,    "The steps interval to print loss.")
+log_g.add_arg("verbose",             bool,   False, "Whether to output verbose log.")
+
+data_g = ArgumentGroup(parser, "data", "Data paths, vocab paths and data processing options")
+data_g.add_arg("train_file",                str,   None,  "SQuAD json for training. E.g., train-v1.1.json.")
+data_g.add_arg("predict_file",              str,   None,  "SQuAD json for predictions. E.g. dev-v1.1.json or test-v1.1.json.")
+data_g.add_arg("vocab_path",                str,   None,  "Vocabulary path.")
+data_g.add_arg("version_2_with_negative",   bool,  False,
+               "If true, the SQuAD examples contain some that do not have an answer. If using squad v2.0, it should be set true.")
+data_g.add_arg("max_seq_len",               int,   512,   "Number of words of the longest seqence.")
+data_g.add_arg("max_query_length",          int,   64,   "Max query length.")
+data_g.add_arg("max_answer_length",         int,   64,   "Max answer length.")
+data_g.add_arg("batch_size",                int,   12,  "Total samples' number in batch for training. see also --in_tokens.")
+data_g.add_arg("in_tokens",                 bool,  False,
+               "If set, the batch size will be the maximum number of tokens in one batch. "
+               "Otherwise, it will be the maximum number of examples in one batch.")
+data_g.add_arg("do_lower_case",             bool,  True,
+               "Whether to lower case the input text. Should be True for uncased models and False for cased models.")
+data_g.add_arg("doc_stride",                int,   128,
+               "When splitting up a long document into chunks, how much stride to take between chunks.")
+data_g.add_arg("n_best_size",               int,   20,
+               "The total number of n-best predictions to generate in the nbest_predictions.json output file.")
+data_g.add_arg("null_score_diff_threshold", float, 0.0,
+               "If null_score - best_non_null is greater than the threshold predict null.")
+data_g.add_arg("random_seed",               int,   0,  "Random seed.")
+
+run_type_g = ArgumentGroup(parser, "run_type", "running type options.")
+run_type_g.add_arg("use_cuda",              bool,   True,  "If set, use GPU for training.")
+run_type_g.add_arg("use_fast_executor",     bool,   False, "If set, use fast parallel executor (in experiment).")
+run_type_g.add_arg("do_train",              bool,   True,  "Whether to perform training.")
+run_type_g.add_arg("do_predict",            bool,   True,  "Whether to perform prediction.")
+
+args = parser.parse_args()
+# yapf: enable.
+
+def create_model(pyreader_name, bert_config, is_training=False):
+    if is_training:
+        pyreader = fluid.layers.py_reader(
+            capacity=50,
+            shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1],
+                    [-1, args.max_seq_len, 1],
+                    [-1, args.max_seq_len, args.max_seq_len], [-1, 1], [-1, 1],
+                    [-1, 1]],
+            dtypes=[
+                'int64', 'int64', 'int64', 'float', 'int64', 'int64', 'int64'
+            ],
+            lod_levels=[0, 0, 0, 0, 0, 0, 0],
+            name=pyreader_name,
+            use_double_buffer=True)
+        (src_ids, pos_ids, sent_ids, self_attn_mask, start_positions,
+         end_positions, next_sent_index) = fluid.layers.read_file(pyreader)
+    else:
+        pyreader = fluid.layers.py_reader(
+            capacity=50,
+            shapes=[[-1, args.max_seq_len, 1], [-1, args.max_seq_len, 1],
+                    [-1, args.max_seq_len, 1],
+                    [-1, args.max_seq_len, args.max_seq_len], [-1, 1], [-1, 1]],
+            dtypes=['int64', 'int64', 'int64', 'float', 'int64', 'int64'],
+            lod_levels=[0, 0, 0, 0, 0, 0],
+            name=pyreader_name,
+            use_double_buffer=True)
+        (src_ids, pos_ids, sent_ids, self_attn_mask, unique_id,
+         next_sent_index) = fluid.layers.read_file(pyreader)
+
+    bert = BertModel(
+        src_ids=src_ids,
+        position_ids=pos_ids,
+        sentence_ids=sent_ids,
+        self_attn_mask=self_attn_mask,
+        config=bert_config,
+        use_fp16=args.use_fp16)
+
+    enc_out = bert.get_sequence_output()
+
+    logits = fluid.layers.fc(
+        input=enc_out,
+        size=2,
+        num_flatten_dims=2,
+        param_attr=fluid.ParamAttr(
+            name="cls_squad_out_w",
+            initializer=fluid.initializer.TruncatedNormal(scale=0.02)),
+        bias_attr=fluid.ParamAttr(
+            name="cls_squad_out_b", initializer=fluid.initializer.Constant(0.)))
+
+    logits = fluid.layers.transpose(x=logits, perm=[2, 0, 1])
+    start_logits, end_logits = fluid.layers.unstack(x=logits, axis=0)
+
+    batch_ones = fluid.layers.fill_constant_batch_size_like(
+        input=start_logits, dtype='int64', shape=[1], value=1)
+    num_seqs = fluid.layers.reduce_sum(input=batch_ones)
+    num_seqs.persistable = True
+    start_logits.persistable = True
+    end_logits.persistable = True
+
+    if is_training:
+
+        def compute_loss(logits, positions):
+            loss = fluid.layers.softmax_with_cross_entropy(
+                logits=logits, label=positions)
+            loss = fluid.layers.mean(x=loss)
+            return loss
+
+        start_loss = compute_loss(start_logits, start_positions)
+        end_loss = compute_loss(end_logits, end_positions)
+        total_loss = (start_loss + end_loss) / 2.0
+        if args.use_fp16 and args.loss_scaling > 1.0:
+            total_loss = total_loss * args.loss_scaling
+        total_loss.persistable = True
+
+        return pyreader, total_loss, num_seqs
+    else:
+        return pyreader, unique_id, start_logits, end_logits, num_seqs
+
+
+RawResult = collections.namedtuple("RawResult",
+                                   ["unique_id", "start_logits", "end_logits"])
+
+
+def predict(test_exe, test_program, test_pyreader, fetch_list, processor):
+    if not os.path.exists(args.checkpoints):
+        os.makedirs(args.checkpoints)
+    output_prediction_file = os.path.join(args.checkpoints, "predictions.json")
+    output_nbest_file = os.path.join(args.checkpoints, "nbest_predictions.json")
+    output_null_log_odds_file = os.path.join(args.checkpoints, "null_odds.json")
+
+    test_pyreader.start()
+    all_results = []
+    time_begin = time.time()
+    while True:
+        try:
+            np_unique_ids, np_start_logits, np_end_logits, np_num_seqs = test_exe.run(
+                fetch_list=fetch_list, program=test_program)
+            for idx in range(np_unique_ids.shape[0]):
+                if len(all_results) % 1000 == 0:
+                    print("Processing example: %d" % len(all_results))
+                unique_id = int(np_unique_ids[idx])
+                start_logits = [float(x) for x in np_start_logits[idx].flat]
+                end_logits = [float(x) for x in np_end_logits[idx].flat]
+                all_results.append(
+                    RawResult(
+                        unique_id=unique_id,
+                        start_logits=start_logits,
+                        end_logits=end_logits))
+        except fluid.core.EOFException:
+            test_pyreader.reset()
+            break
+    time_end = time.time()
+
+    features = processor.get_features(
+        processor.predict_examples, is_training=False)
+    write_predictions(processor.predict_examples, features, all_results,
+                      args.n_best_size, args.max_answer_length,
+                      args.do_lower_case, output_prediction_file,
+                      output_nbest_file, output_null_log_odds_file,
+                      args.version_2_with_negative,
+                      args.null_score_diff_threshold, args.verbose)
+
+
+def train(args):
+    bert_config = BertConfig(args.bert_config_path)
+    bert_config.print_config()
+
+    if not (args.do_train or args.do_predict):
+        raise ValueError("For args `do_train` and `do_predict`, at "
+                         "least one of them must be True.")
+
+    if args.use_cuda:
+        place = fluid.CUDAPlace(0)
+        dev_count = fluid.core.get_cuda_device_count()
+    else:
+        place = fluid.CPUPlace()
+        dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count()))
+    exe = fluid.Executor(place)
+
+    processor = DataProcessor(
+        vocab_path=args.vocab_path,
+        do_lower_case=args.do_lower_case,
+        max_seq_length=args.max_seq_len,
+        in_tokens=args.in_tokens,
+        doc_stride=args.doc_stride,
+        max_query_length=args.max_query_length)
+
+    startup_prog = fluid.Program()
+    if args.random_seed is not None:
+        startup_prog.random_seed = args.random_seed
+
+    if args.do_train:
+        train_data_generator = processor.data_generator(
+            data_path=args.train_file,
+            batch_size=args.batch_size,
+            phase='train',
+            shuffle=False,
+            version_2_with_negative=args.version_2_with_negative,
+            epoch=args.epoch)
+
+        num_train_examples = processor.get_num_examples(phase='train')
+        if args.in_tokens:
+            max_train_steps = args.epoch * num_train_examples // (
+                args.batch_size // args.max_seq_len) // dev_count
+        else:
+            max_train_steps = args.epoch * num_train_examples // (
+                args.batch_size) // dev_count
+        warmup_steps = int(max_train_steps * args.warmup_proportion)
+        print("Device count: %d" % dev_count)
+        print("Num train examples: %d" % num_train_examples)
+        print("Max train steps: %d" % max_train_steps)
+        print("Num warmup steps: %d" % warmup_steps)
+
+        train_program = fluid.Program()
+        with fluid.program_guard(train_program, startup_prog):
+            with fluid.unique_name.guard():
+                train_pyreader, loss, num_seqs = create_model(
+                    pyreader_name='train_reader',
+                    bert_config=bert_config,
+                    is_training=True)
+
+                scheduled_lr = optimization(
+                    loss=loss,
+                    warmup_steps=warmup_steps,
+                    num_train_steps=max_train_steps,
+                    learning_rate=args.learning_rate,
+                    train_program=train_program,
+                    startup_prog=startup_prog,
+                    weight_decay=args.weight_decay,
+                    scheduler=args.lr_scheduler,
+                    use_fp16=args.use_fp16,
+                    loss_scaling=args.loss_scaling)
+
+                fluid.memory_optimize(train_program)
+
+        if args.verbose:
+            if args.in_tokens:
+                lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
+                    program=train_program,
+                    batch_size=args.batch_size // args.max_seq_len)
+            else:
+                lower_mem, upper_mem, unit = fluid.contrib.memory_usage(
+                    program=train_program, batch_size=args.batch_size)
+            print("Theoretical memory usage in training:  %.3f - %.3f %s" %
+                  (lower_mem, upper_mem, unit))
+
+    if args.do_predict:
+        test_prog = fluid.Program()
+        with fluid.program_guard(test_prog, startup_prog):
+            with fluid.unique_name.guard():
+                test_pyreader, unique_ids, start_logits, end_logits, num_seqs = create_model(
+                    pyreader_name='test_reader',
+                    bert_config=bert_config,
+                    is_training=False)
+
+                fluid.memory_optimize(test_prog)
+
+        test_prog = test_prog.clone(for_test=True)
+
+    exe.run(startup_prog)
+
+    if args.do_train:
+        if args.init_checkpoint and args.init_pretraining_params:
+            print(
+                "WARNING: args 'init_checkpoint' and 'init_pretraining_params' "
+                "both are set! Only arg 'init_checkpoint' is made valid.")
+        if args.init_checkpoint:
+            init_checkpoint(
+                exe,
+                args.init_checkpoint,
+                main_program=startup_prog,
+                use_fp16=args.use_fp16)
+        elif args.init_pretraining_params:
+            init_pretraining_params(
+                exe,
+                args.init_pretraining_params,
+                main_program=startup_prog,
+                use_fp16=args.use_fp16)
+    elif args.do_predict:
+        if not args.init_checkpoint:
+            raise ValueError("args 'init_checkpoint' should be set if"
+                             "only doing prediction!")
+        init_checkpoint(
+            exe,
+            args.init_checkpoint,
+            main_program=startup_prog,
+            use_fp16=args.use_fp16)
+
+    if args.do_train:
+        exec_strategy = fluid.ExecutionStrategy()
+        if args.use_fast_executor:
+            exec_strategy.use_experimental_executor = True
+        exec_strategy.num_threads = dev_count
+        exec_strategy.num_iteration_per_drop_scope = min(10, args.skip_steps)
+
+        train_exe = fluid.ParallelExecutor(
+            use_cuda=args.use_cuda,
+            loss_name=loss.name,
+            exec_strategy=exec_strategy,
+            main_program=train_program)
+
+        train_pyreader.decorate_tensor_provider(train_data_generator)
+
+        train_pyreader.start()
+        steps = 0
+        total_cost, total_num_seqs = [], []
+        time_begin = time.time()
+        while steps < max_train_steps:
+            try:
+                steps += 1
+                if steps % args.skip_steps == 0:
+                    if warmup_steps <= 0:
+                        fetch_list = [loss.name, num_seqs.name]
+                    else:
+                        fetch_list = [
+                            loss.name, scheduled_lr.name, num_seqs.name
+                        ]
+                else:
+                    fetch_list = []
+
+                outputs = train_exe.run(fetch_list=fetch_list)
+
+                if steps % args.skip_steps == 0:
+                    if warmup_steps <= 0:
+                        np_loss, np_num_seqs = outputs
+                    else:
+                        np_loss, np_lr, np_num_seqs = outputs
+                    total_cost.extend(np_loss * np_num_seqs)
+                    total_num_seqs.extend(np_num_seqs)
+
+                    if args.verbose:
+                        verbose = "train pyreader queue size: %d, " % train_pyreader.queue.size(
+                        )
+                        verbose += "learning rate: %f" % (
+                            np_lr[0]
+                            if warmup_steps > 0 else args.learning_rate)
+                        print(verbose)
+
+                    time_end = time.time()
+                    used_time = time_end - time_begin
+                    current_example, epoch = processor.get_train_progress()
+
+                    print("epoch: %d, progress: %d/%d, step: %d, loss: %f, "
+                          "speed: %f steps/s" %
+                          (epoch, current_example, num_train_examples, steps,
+                           np.sum(total_cost) / np.sum(total_num_seqs),
+                           args.skip_steps / used_time))
+                    total_cost, total_num_seqs = [], []
+                    time_begin = time.time()
+
+                if steps % args.save_steps == 0:
+                    save_path = os.path.join(args.checkpoints,
+                                             "step_" + str(steps))
+                    fluid.io.save_persistables(exe, save_path, train_program)
+            except fluid.core.EOFException:
+                save_path = os.path.join(args.checkpoints,
+                                         "step_" + str(steps) + "_final")
+                fluid.io.save_persistables(exe, save_path, train_program)
+                train_pyreader.reset()
+                break
+
+    if args.do_predict:
+        test_pyreader.decorate_tensor_provider(
+            processor.data_generator(
+                data_path=args.predict_file,
+                batch_size=args.batch_size,
+                phase='predict',
+                shuffle=False,
+                epoch=1))
+
+        predict(exe, test_prog, test_pyreader, [
+            unique_ids.name, start_logits.name, end_logits.name, num_seqs.name
+        ], processor)
+
+
+if __name__ == '__main__':
+    print_arguments(args)
+    train(args)

BERT/test_local_dist.sh

+#!/bin/bash
+set -xe
+
+# Paddle debug envs
+export GLOG_v=1
+export GLOG_logtostderr=1
+
+# Unset proxy
+unset https_proxy http_proxy
+
+# NCCL debug envs
+export NCCL_P2P_DISABLE=1
+export NCCL_DEBUG=INFO
+# Comment it if your nccl support IB
+export NCCL_IB_DISABLE=1
+
+# Add your nodes endpoints here.
+export worker_endpoints=127.0.0.1:9184,127.0.0.1:9185
+export current_endpoint=127.0.0.1:9184
+export CUDA_VISIBLE_DEVICES=0
+
+./train.sh -local n > 0.log 2>&1 &
+
+# Add your nodes endpoints here.
+export current_endpoint=127.0.0.1:9185
+export CUDA_VISIBLE_DEVICES=1
+
+./train.sh -local n > 1.log 2>&1 &

BERT/tokenization.py

+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import unicodedata
+import six
+
+
+def convert_to_unicode(text):
+    """Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
+    if six.PY3:
+        if isinstance(text, str):
+            return text
+        elif isinstance(text, bytes):
+            return text.decode("utf-8", "ignore")
+        else:
+            raise ValueError("Unsupported string type: %s" % (type(text)))
+    elif six.PY2:
+        if isinstance(text, str):
+            return text.decode("utf-8", "ignore")
+        elif isinstance(text, unicode):
+            return text
+        else:
+            raise ValueError("Unsupported string type: %s" % (type(text)))
+    else:
+        raise ValueError("Not running on Python2 or Python 3?")
+
+
+def printable_text(text):
+    """Returns text encoded in a way suitable for print or `tf.logging`."""
+
+    # These functions want `str` for both Python2 and Python3, but in one case
+    # it's a Unicode string and in the other it's a byte string.
+    if six.PY3:
+        if isinstance(text, str):
+            return text
+        elif isinstance(text, bytes):
+            return text.decode("utf-8", "ignore")
+        else:
+            raise ValueError("Unsupported string type: %s" % (type(text)))
+    elif six.PY2:
+        if isinstance(text, str):
+            return text
+        elif isinstance(text, unicode):
+            return text.encode("utf-8")
+        else:
+            raise ValueError("Unsupported string type: %s" % (type(text)))
+    else:
+        raise ValueError("Not running on Python2 or Python 3?")
+
+
+def load_vocab(vocab_file):
+    """Loads a vocabulary file into a dictionary."""
+    vocab = collections.OrderedDict()
+    fin = open(vocab_file)
+    for num, line in enumerate(fin):
+        items = convert_to_unicode(line.strip()).split("\t")
+        if len(items) > 2:
+            break
+        token = items[0]
+        index = items[1] if len(items) == 2 else num
+        token = token.strip()
+        vocab[token] = int(index)
+    return vocab
+
+
+def convert_by_vocab(vocab, items):
+    """Converts a sequence of [tokens|ids] using the vocab."""
+    output = []
+    for item in items:
+        output.append(vocab[item])
+    return output
+
+
+def convert_tokens_to_ids(vocab, tokens):
+    return convert_by_vocab(vocab, tokens)
+
+
+def convert_ids_to_tokens(inv_vocab, ids):
+    return convert_by_vocab(inv_vocab, ids)
+
+
+def whitespace_tokenize(text):
+    """Runs basic whitespace cleaning and splitting on a peice of text."""
+    text = text.strip()
+    if not text:
+        return []
+    tokens = text.split()
+    return tokens
+
+
+class FullTokenizer(object):
+    """Runs end-to-end tokenziation."""
+
+    def __init__(self, vocab_file, do_lower_case=True):
+        self.vocab = load_vocab(vocab_file)
+        self.inv_vocab = {v: k for k, v in self.vocab.items()}
+        self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)
+        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+
+    def tokenize(self, text):
+        split_tokens = []
+        for token in self.basic_tokenizer.tokenize(text):
+            for sub_token in self.wordpiece_tokenizer.tokenize(token):
+                split_tokens.append(sub_token)
+
+        return split_tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return convert_by_vocab(self.vocab, tokens)
+
+    def convert_ids_to_tokens(self, ids):
+        return convert_by_vocab(self.inv_vocab, ids)
+
+
+class CharTokenizer(object):
+    """Runs end-to-end tokenziation."""
+
+    def __init__(self, vocab_file, do_lower_case=True):
+        self.vocab = load_vocab(vocab_file)
+        self.inv_vocab = {v: k for k, v in self.vocab.items()}
+        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)
+
+    def tokenize(self, text):
+        split_tokens = []
+        for token in text.lower().split(" "):
+            for sub_token in self.wordpiece_tokenizer.tokenize(token):
+                split_tokens.append(sub_token)
+
+        return split_tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        return convert_by_vocab(self.vocab, tokens)
+
+    def convert_ids_to_tokens(self, ids):
+        return convert_by_vocab(self.inv_vocab, ids)
+
+
+class BasicTokenizer(object):
+    """Runs basic tokenization (punctuation splitting, lower casing, etc.)."""
+
+    def __init__(self, do_lower_case=True):
+        """Constructs a BasicTokenizer.
+
+        Args:
+            do_lower_case: Whether to lower case the input.
+        """
+        self.do_lower_case = do_lower_case
+
+    def tokenize(self, text):
+        """Tokenizes a piece of text."""
+        text = convert_to_unicode(text)
+        text = self._clean_text(text)
+
+        # This was added on November 1st, 2018 for the multilingual and Chinese
+        # models. This is also applied to the English models now, but it doesn't
+        # matter since the English models were not trained on any Chinese data
+        # and generally don't have any Chinese data in them (there are Chinese
+        # characters in the vocabulary because Wikipedia does have some Chinese
+        # words in the English Wikipedia.).
+        text = self._tokenize_chinese_chars(text)
+
+        orig_tokens = whitespace_tokenize(text)
+        split_tokens = []
+        for token in orig_tokens:
+            if self.do_lower_case:
+                token = token.lower()
+                token = self._run_strip_accents(token)
+            split_tokens.extend(self._run_split_on_punc(token))
+
+        output_tokens = whitespace_tokenize(" ".join(split_tokens))
+        return output_tokens
+
+    def _run_strip_accents(self, text):
+        """Strips accents from a piece of text."""
+        text = unicodedata.normalize("NFD", text)
+        output = []
+        for char in text:
+            cat = unicodedata.category(char)
+            if cat == "Mn":
+                continue
+            output.append(char)
+        return "".join(output)
+
+    def _run_split_on_punc(self, text):
+        """Splits punctuation on a piece of text."""
+        chars = list(text)
+        i = 0
+        start_new_word = True
+        output = []
+        while i < len(chars):
+            char = chars[i]
+            if _is_punctuation(char):
+                output.append([char])
+                start_new_word = True
+            else:
+                if start_new_word:
+                    output.append([])
+                start_new_word = False
+                output[-1].append(char)
+            i += 1
+
+        return ["".join(x) for x in output]
+
+    def _tokenize_chinese_chars(self, text):
+        """Adds whitespace around any CJK character."""
+        output = []
+        for char in text:
+            cp = ord(char)
+            if self._is_chinese_char(cp):
+                output.append(" ")
+                output.append(char)
+                output.append(" ")
+            else:
+                output.append(char)
+        return "".join(output)
+
+    def _is_chinese_char(self, cp):
+        """Checks whether CP is the codepoint of a CJK character."""
+        # This defines a "chinese character" as anything in the CJK Unicode block:
+        #     https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
+        #
+        # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
+        # despite its name. The modern Korean Hangul alphabet is a different block,
+        # as is Japanese Hiragana and Katakana. Those alphabets are used to write
+        # space-separated words, so they are not treated specially and handled
+        # like the all of the other languages.
+        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #
+            (cp >= 0x3400 and cp <= 0x4DBF) or  #
+            (cp >= 0x20000 and cp <= 0x2A6DF) or  #
+            (cp >= 0x2A700 and cp <= 0x2B73F) or  #
+            (cp >= 0x2B740 and cp <= 0x2B81F) or  #
+            (cp >= 0x2B820 and cp <= 0x2CEAF) or
+            (cp >= 0xF900 and cp <= 0xFAFF) or  #
+            (cp >= 0x2F800 and cp <= 0x2FA1F)):  #
+            return True
+
+        return False
+
+    def _clean_text(self, text):
+        """Performs invalid character removal and whitespace cleanup on text."""
+        output = []
+        for char in text:
+            cp = ord(char)
+            if cp == 0 or cp == 0xfffd or _is_control(char):
+                continue
+            if _is_whitespace(char):
+                output.append(" ")
+            else:
+                output.append(char)
+        return "".join(output)
+
+
+class WordpieceTokenizer(object):
+    """Runs WordPiece tokenziation."""
+
+    def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=100):
+        self.vocab = vocab
+        self.unk_token = unk_token
+        self.max_input_chars_per_word = max_input_chars_per_word
+
+    def tokenize(self, text):
+        """Tokenizes a piece of text into its word pieces.
+
+        This uses a greedy longest-match-first algorithm to perform tokenization
+        using the given vocabulary.
+
+        For example:
+            input = "unaffable"
+            output = ["un", "##aff", "##able"]
+
+        Args:
+            text: A single token or whitespace separated tokens. This should have
+                already been passed through `BasicTokenizer.
+
+        Returns:
+            A list of wordpiece tokens.
+        """
+
+        text = convert_to_unicode(text)
+
+        output_tokens = []
+        for token in whitespace_tokenize(text):
+            chars = list(token)
+            if len(chars) > self.max_input_chars_per_word:
+                output_tokens.append(self.unk_token)
+                continue
+
+            is_bad = False
+            start = 0
+            sub_tokens = []
+            while start < len(chars):
+                end = len(chars)
+                cur_substr = None
+                while start < end:
+                    substr = "".join(chars[start:end])
+                    if start > 0:
+                        substr = "##" + substr
+                    if substr in self.vocab:
+                        cur_substr = substr
+                        break
+                    end -= 1
+                if cur_substr is None:
+                    is_bad = True
+                    break
+                sub_tokens.append(cur_substr)
+                start = end
+
+            if is_bad:
+                output_tokens.append(self.unk_token)
+            else:
+                output_tokens.extend(sub_tokens)
+        return output_tokens
+
+
+def _is_whitespace(char):
+    """Checks whether `chars` is a whitespace character."""
+    # \t, \n, and \r are technically contorl characters but we treat them
+    # as whitespace since they are generally considered as such.
+    if char == " " or char == "\t" or char == "\n" or char == "\r":
+        return True
+    cat = unicodedata.category(char)
+    if cat == "Zs":
+        return True
+    return False
+
+
+def _is_control(char):
+    """Checks whether `chars` is a control character."""
+    # These are technically control characters but we count them as whitespace
+    # characters.
+    if char == "\t" or char == "\n" or char == "\r":
+        return False
+    cat = unicodedata.category(char)
+    if cat.startswith("C"):
+        return True
+    return False
+
+
+def _is_punctuation(char):
+    """Checks whether `chars` is a punctuation character."""
+    cp = ord(char)
+    # We treat all non-letter/number ASCII as punctuation.
+    # Characters such as "^", "$", and "`" are not in the Unicode
+    # Punctuation class but we treat them as punctuation anyways, for
+    # consistency.
+    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or
+        (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):
+        return True
+    cat = unicodedata.category(char)
+    if cat.startswith("P"):
+        return True
+    return False

BERT/train.sh

+#!/bin/bash
+
+set -xe
+
+while true ; do
+  case "$1" in
+    -local) is_local="$2" ; shift 2 ;;
+    *)
+       if [[ ${#1} > 0 ]]; then
+          echo "not supported arugments ${1}" ; exit 1 ;
+       else
+           break
+       fi
+       ;;
+  esac
+done
+
+case "$is_local" in
+    n) is_distributed="--is_distributed true" ;;
+    y) is_distributed="--is_distributed false" ;;
+    *) echo "not support argument -local: ${is_local}" ; exit 1 ;;
+esac
+
+# pretrain config
+SAVE_STEPS=10000
+BATCH_SIZE=4096
+LR_RATE=1e-4
+WEIGHT_DECAY=0.01
+MAX_LEN=512
+TRAIN_DATA_DIR=data/train
+VALIDATION_DATA_DIR=data/validation
+CONFIG_PATH=config/bert_config.json
+VOCAB_PATH=config/vocab.txt
+
+# Change your train arguments:
+python -u ./train.py ${is_distributed}\
+        --use_cuda true\
+        --weight_sharing true\
+        --batch_size ${BATCH_SIZE} \
+        --data_dir ${TRAIN_DATA_DIR} \
+        --validation_set_dir ${VALIDATION_DATA_DIR} \
+        --bert_config_path ${CONFIG_PATH} \
+        --vocab_path ${VOCAB_PATH} \
+        --generate_neg_sample true\
+        --checkpoints ./output \
+        --save_steps ${SAVE_STEPS} \
+        --learning_rate ${LR_RATE} \
+        --weight_decay ${WEIGHT_DECAY:-0} \
+        --max_seq_len ${MAX_LEN} \
+        --skip_steps 20 \
+        --validation_steps 1000

BERT/utils/args.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Arguments for configuration."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six
+import argparse
+
+
+def str2bool(v):
+    # because argparse does not support to parse "true, False" as python
+    # boolean directly
+    return v.lower() in ("true", "t", "1")
+
+
+class ArgumentGroup(object):
+    def __init__(self, parser, title, des):
+        self._group = parser.add_argument_group(title=title, description=des)
+
+    def add_arg(self, name, type, default, help, **kwargs):
+        type = str2bool if type == bool else type
+        self._group.add_argument(
+            "--" + name,
+            default=default,
+            type=type,
+            help=help + ' Default: %(default)s.',
+            **kwargs)
+
+
+def print_arguments(args):
+    print('-----------  Configuration Arguments -----------')
+    for arg, value in sorted(six.iteritems(vars(args))):
+        print('%s: %s' % (arg, value))
+    print('------------------------------------------------')

BERT/utils/fp16.py

+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import paddle
+import paddle.fluid as fluid
+
+
+def cast_fp16_to_fp32(i, o, prog):
+    prog.global_block().append_op(
+        type="cast",
+        inputs={"X": i},
+        outputs={"Out": o},
+        attrs={
+            "in_dtype": fluid.core.VarDesc.VarType.FP16,
+            "out_dtype": fluid.core.VarDesc.VarType.FP32
+        })
+
+
+def cast_fp32_to_fp16(i, o, prog):
+    prog.global_block().append_op(
+        type="cast",
+        inputs={"X": i},
+        outputs={"Out": o},
+        attrs={
+            "in_dtype": fluid.core.VarDesc.VarType.FP32,
+            "out_dtype": fluid.core.VarDesc.VarType.FP16
+        })
+
+
+def copy_to_master_param(p, block):
+    v = block.vars.get(p.name, None)
+    if v is None:
+        raise ValueError("no param name %s found!" % p.name)
+    new_p = fluid.framework.Parameter(
+        block=block,
+        shape=v.shape,
+        dtype=fluid.core.VarDesc.VarType.FP32,
+        type=v.type,
+        lod_level=v.lod_level,
+        stop_gradient=p.stop_gradient,
+        trainable=p.trainable,
+        optimize_attr=p.optimize_attr,
+        regularizer=p.regularizer,
+        gradient_clip_attr=p.gradient_clip_attr,
+        error_clip=p.error_clip,
+        name=v.name + ".master")
+    return new_p
+
+
+def create_master_params_grads(params_grads, main_prog, startup_prog,
+                               loss_scaling):
+    master_params_grads = []
+    tmp_role = main_prog._current_role
+    OpRole = fluid.core.op_proto_and_checker_maker.OpRole
+    main_prog._current_role = OpRole.Backward
+    for p, g in params_grads:
+        # create master parameters
+        master_param = copy_to_master_param(p, main_prog.global_block())
+        startup_master_param = startup_prog.global_block()._clone_variable(
+            master_param)
+        startup_p = startup_prog.global_block().var(p.name)
+        cast_fp16_to_fp32(startup_p, startup_master_param, startup_prog)
+        # cast fp16 gradients to fp32 before apply gradients
+        if g.name.find("layer_norm") > -1:
+            if loss_scaling > 1:
+                scaled_g = g / float(loss_scaling)
+            else:
+                scaled_g = g
+            master_params_grads.append([p, scaled_g])
+            continue
+        master_grad = fluid.layers.cast(g, "float32")
+        if loss_scaling > 1:
+            master_grad = master_grad / float(loss_scaling)
+        master_params_grads.append([master_param, master_grad])
+    main_prog._current_role = tmp_role
+    return master_params_grads
+
+
+def master_param_to_train_param(master_params_grads, params_grads, main_prog):
+    for idx, m_p_g in enumerate(master_params_grads):
+        train_p, _ = params_grads[idx]
+        if train_p.name.find("layer_norm") > -1:
+            continue
+        with main_prog._optimized_guard([m_p_g[0], m_p_g[1]]):
+            cast_fp32_to_fp16(m_p_g[0], train_p, main_prog)

README.md

 # LARK
 
-**LA**nguage **R**epresentations **K**it, currently including
+**LA**nguage **R**epresentations **K**it, currently includes
 
-[BERT](./BERT): Bidirectional Encoder Representation from Transformers.
+- [BERT](./BERT): Bidirectional Encoder Representation from Transformers
 
 
 And more is on the way.