fix bugs and refine reader name

.gitignore

@@ -2,5 +2,8 @@
 __pycache__
 pretrain_model
 output_model
+build
+dist
+paddle_palm.egg-info
 mrqa_output
 *.log

config_demo1.yaml

@@ -5,10 +5,6 @@ save_path: "output_model/firstrun"
 backbone: "bert"
 backbone_config_path: "pretrain_model/bert/bert_config.json"
 
-vocab_path: "pretrain_model/bert/vocab.txt"
-do_lower_case: True
-max_seq_len: 512
-
 batch_size: 5
 num_epochs: 3
 optimizer: "adam"

config_demo3.py

+task_instance: "mrqa"
+
+save_path: "output_model/firstrun"
+
+backbone: "bert"
+backbone_config_path: "pretrain_model/bert/bert_config.json"
+
+vocab_path: "pretrain_model/bert/vocab.txt"
+do_lower_case: True
+max_seq_len: 512
+
+batch_size: 5
+num_epochs: 3
+optimizer: "adam"
+learning_rate: 3e-5
+warmup_proportion: 0.1
+weight_decay: 0.1
+
+print_every_n_steps: 1

demo1.py

@@ -5,6 +5,3 @@ if __name__ == '__main__':
     controller.load_pretrain('pretrain_model/bert/params')
     controller.train()
 
-    controller = palm.Controller(config='config_demo1.yaml', task_dir='demo1_tasks', for_train=False)
-    controller.pred('mrqa', inference_model_dir='output_model/firstrun/infer_model')
-

demo1_tasks/mrqa.yaml

-train_file: data/mrqa/mrqa-combined.train.raw.json
-pred_file: data/mrqa/mrqa-combined.dev.raw.json
-pred_output_path: 'mrqa_output'
-reader: mrc4ernie
+train_file: data/mrqa/train.json
+reader: mrc
 paradigm: mrc
+
+vocab_path: "pretrain_model/bert/vocab.txt"
+do_lower_case: True
+max_seq_len: 512
 doc_stride: 128
 max_query_len: 64
-max_answer_len: 30
-n_best_size: 20
-null_score_diff_threshold: 0.0
-verbose: False

demo2.py

@@ -3,8 +3,9 @@ import paddlepalm as palm
 if __name__ == '__main__':
     controller = palm.Controller('config_demo2.yaml', task_dir='demo2_tasks')
     controller.load_pretrain('pretrain_model/ernie/params')
-    # controller.train()
+    controller.train()
 
     # controller = palm.Controller(config='config_demo2.yaml', task_dir='demo2_tasks', for_train=False)
     # controller.pred('mrqa', inference_model_dir='output_model/secondrun/infer_model')
 
+

demo2_tasks/match4mrqa.yaml

-train_file: "data/match4mrqa/train.txt"
-reader: match4ernie
+train_file: "data/match4mrqa/train.tsv"
+reader: match
 paradigm: match
 

demo2_tasks/mlm4mrqa.yaml

-train_file: "data/mlm4mrqa/train.txt"
+train_file: "data/mlm4mrqa/train.tsv"
 reader: mlm
 paradigm: mlm

demo2_tasks/mrqa.yaml

-train_file: data/mrqa/mrqa-combined.train.raw.json
-pred_file: data/mrqa/mrqa-combined.dev.raw.json
+train_file: data/mrqa/train.json
+pred_file: data/mrqa/dev.json
 pred_output_path: 'mrqa_output'
-reader: mrc4ernie
+reader: mrc
 paradigm: mrc
 doc_stride: 128
 max_query_len: 64

demo3.py

+import paddlepalm as palm
+
+if __name__ == '__main__':
+    controller = palm.Controller('config_demo3.yaml', task_dir='demo3_tasks')
+    controller.load_pretrain('pretrain_model/ernie/params')
+    controller.train()
+
+    controller = palm.Controller(config='config_demo3.yaml', task_dir='demo3_tasks', for_train=False)
+    controller.pred('cls4mrqa', inference_model_dir='output_model/thirdrun/infer_model')
+

demo3_tasks/mrqa.yaml

+train_file: data/cls4mrqa/train.tsv
+reader: cls
+paradigm: cls

nohup.out

+W1028 21:51:59.319365  9630 device_context.cc:235] Please NOTE: device: 0, CUDA Capability: 61, Driver API Version: 10.1, Runtime API Version: 9.0
+W1028 21:51:59.323333  9630 device_context.cc:243] device: 0, cuDNN Version: 7.3.
+I1028 21:52:26.817137  9630 parallel_executor.cc:421] The number of CUDAPlace, which is used in ParallelExecutor, is 8. And the Program will be copied 8 copies
+W1028 21:52:41.982228  9630 fuse_all_reduce_op_pass.cc:72] Find all_reduce operators: 401. To make the speed faster, some all_reduce ops are fused during training, after fusion, the number of all_reduce ops is 255.
+I1028 21:52:42.243458  9630 build_strategy.cc:363] SeqOnlyAllReduceOps:0, num_trainers:1
+I1028 21:53:14.242537  9630 parallel_executor.cc:285] Inplace strategy is enabled, when build_strategy.enable_inplace = True
+I1028 21:53:16.313246  9630 parallel_executor.cc:368] Garbage collection strategy is enabled, when FLAGS_eager_delete_tensor_gb = 0
+/home/zhangyiming/env-bert/lib/python2.7/site-packages/paddle/fluid/executor.py:774: UserWarning: The following exception is not an EOF exception.
+  "The following exception is not an EOF exception.")
+Traceback (most recent call last):
+  File "demo2.py", line 6, in <module>
+    controller.train()
+  File "/home/ssd7/yiming/release/PALM/paddlepalm/mtl_controller.py", line 669, in train
+    fluid.io.save_persistables(self.exe, save_path, saver_program)
+  File "/home/zhangyiming/env-bert/lib/python2.7/site-packages/paddle/fluid/io.py", line 571, in save_persistables
+    filename=filename)
+  File "/home/zhangyiming/env-bert/lib/python2.7/site-packages/paddle/fluid/io.py", line 216, in save_vars
+    filename=filename)
+  File "/home/zhangyiming/env-bert/lib/python2.7/site-packages/paddle/fluid/io.py", line 256, in save_vars
+    executor.run(save_program)
+  File "/home/zhangyiming/env-bert/lib/python2.7/site-packages/paddle/fluid/executor.py", line 775, in run
+    six.reraise(*sys.exc_info())
+  File "/home/zhangyiming/env-bert/lib/python2.7/site-packages/paddle/fluid/executor.py", line 770, in run
+    use_program_cache=use_program_cache)
+  File "/home/zhangyiming/env-bert/lib/python2.7/site-packages/paddle/fluid/executor.py", line 817, in _run_impl
+    use_program_cache=use_program_cache)
+  File "/home/zhangyiming/env-bert/lib/python2.7/site-packages/paddle/fluid/executor.py", line 894, in _run_program
+    fetch_var_name)
+paddle.fluid.core_avx.EnforceNotMet: 
+
+--------------------------------------------
+C++ Call Stacks (More useful to developers):
+--------------------------------------------
+0   std::string paddle::platform::GetTraceBackString<char const*>(char const*&&, char const*, int)
+1   paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int)
+2   paddle::operators::SaveOpKernel<paddle::platform::CUDADeviceContext, float>::SaveLodTensor(paddle::framework::ExecutionContext const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::Variable const*) const
+3   paddle::operators::SaveOpKernel<paddle::platform::CUDADeviceContext, float>::Compute(paddle::framework::ExecutionContext const&) const
+4   std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 0ul, paddle::operators::SaveOpKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::SaveOpKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::SaveOpKernel<paddle::platform:I1029 10:38:26.419725 30194 parallel_executor.cc:421] The number of CUDAPlace, which is used in ParallelExecutor, is 8. And the Program will be copied 8 copies
+W1029 10:38:48.046470 30194 fuse_all_reduce_op_pass.cc:72] Find all_reduce operators: 401. To make the speed faster, some all_reduce ops are fused during training, after fusion, the number of all_reduce ops is 255.
+I1029 10:38:48.322405 30194 build_strategy.cc:363] SeqOnlyAllReduceOps:0, num_trainers:1
+I1029 10:39:23.302821 30194 parallel_executor.cc:285] Inplace strategy is enabled, when build_strategy.enable_inplace = True
+I1029 10:39:25.419924 30194 parallel_executor.cc:368] Garbage collection strategy is enabled, when FLAGS_eager_delete_tensor_gb = 0
+W1029 10:42:46.438006 30194 init.cc:212] *** Aborted at 1572316966 (unix time) try "date -d @1572316966" if you are using GNU date ***
+W1029 10:42:46.440183 30194 init.cc:212] PC: @                0x0 (unknown)
+W1029 10:42:46.440296 30194 init.cc:212] *** SIGTERM (@0x1f80000785a) received by PID 30194 (TID 0x7f0773d5e700) from PID 30810; stack trace: ***
+W1029 10:42:46.441951 30194 init.cc:212]     @     0x7f0773528160 (unknown)
+W1029 10:42:46.443789 30194 init.cc:212]     @     0x7f07735243cc __pthread_cond_wait
+W1029 10:42:46.444838 30194 init.cc:212]     @     0x7f0726a0c3cc std::condition_variable::wait()
+W1029 10:42:46.449384 30194 init.cc:212]     @     0x7f070292290d paddle::framework::details::FastThreadedSSAGraphExecutor::Run()
+W1029 10:42:46.450734 30194 init.cc:212]     @     0x7f07028836a7 _ZNSt17_Function_handlerIFvvEZN6paddle9framework7details29ScopeBufferedSSAGraphExecutor3RunERKSt6vectorISsSaISsEEEUlvE_E9_M_invokeERKSt9_Any_data
+W1029 10:42:46.454063 30194 init.cc:212]     @     0x7f07028884bf paddle::framework::details::ScopeBufferedMonitor::Apply()
+W1029 10:42:46.455735 30194 init.cc:212]     @     0x7f0702883e86 paddle::framework::details::ScopeBufferedSSAGraphExecutor::Run()
+W1029 10:42:46.458518 30194 init.cc:212]     @     0x7f0700626038 paddle::framework::ParallelExecutor::Run()
+W1029 10:42:46.459216 30194 init.cc:212]     @     0x7f0700409e78 _ZZN8pybind1112cpp_function10initializeIZN6paddle6pybindL22pybind11_init_core_avxERNS_6moduleEEUlRNS2_9framework16ParallelExecutorERKSt6vectorISsSaISsEEE188_S9_INS6_9LoDTensorESaISF_EEIS8_SD_EINS_4nameENS_9is_methodENS_7siblingEEEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE1_4_FUNESY_
+W1029 10:42:46.460702 30194 init.cc:212]     @     0x7f0700453f56 pybind11::cpp_function::dispatcher()
+W1029 10:42:46.462498 30194 init.cc:212]     @     0x7f0773841cc8 PyEval_EvalFrameEx
+W1029 10:42:46.464206 30194 init.cc:212]     @     0x7f077384435d PyEval_EvalCodeEx
+W1029 10:42:46.465894 30194 init.cc:212]     @     0x7f0773841d50 PyEval_EvalFrameEx
+W1029 10:42:46.467593 30194 init.cc:212]     @     0x7f077384435d PyEval_EvalCodeEx
+W1029 10:42:46.469327 30194 init.cc:212]     @     0x7f0773841d50 PyEval_EvalFrameEx
+W1029 10:42:46.471053 30194 init.cc:212]     @     0x7f077384435d PyEval_EvalCodeEx
+W1029 10:42:46.472759 30194 init.cc:212]     @     0x7f0773841d50 PyEval_EvalFrameEx
+W1029 10:42:46.474479 30194 init.cc:212]     @     0x7f077384435d PyEval_EvalCodeEx
+W1029 10:42:46.476193 30194 init.cc:212]     @     0x7f0773841d50 PyEval_EvalFrameEx
+W1029 10:42:46.477926 30194 init.cc:212]     @     0x7f077384435d PyEval_EvalCodeEx
+W1029 10:42:46.479651 30194 init.cc:212]     @     0x7f0773844492 PyEval_EvalCode
+W1029 10:42:46.481353 30194 init.cc:212]     @     0x7f077386e1a2 PyRun_FileExFlags
+W1029 10:42:46.483080 30194 init.cc:212]     @     0x7f077386f539 PyRun_SimpleFileExFlags
+W1029 10:42:46.484786 30194 init.cc:212]     @     0x7f07738851bd Py_Main
+W1029 10:42:46.487162 30194 init.cc:212]     @     0x7f0772a82bd5 __libc_start_main
+W1029 10:42:46.487229 30194 init.cc:212]     @           0x4007a1 (unknown)
+W1029 10:42:46.488940 30194 init.cc:212]     @                0x0 (unknown)
+./run_demo2.sh: line 5: 30194 Terminated              python demo2.py >> demo2.log
+W1029 10:43:27.495725 32687 device_context.cc:235] Please NOTE: device: 0, CUDA Capability: 61, Driver API Version: 10.1, Runtime API Version: 9.0
+W1029 10:43:27.500324 32687 device_context.cc:243] device: 0, cuDNN Version: 7.3.
+I1029 10:43:41.409127 32687 parallel_executor.cc:421] The number of CUDAPlace, which is used in ParallelExecutor, is 8. And the Program will be copied 8 copies
+W1029 10:44:03.299010 32687 fuse_all_reduce_op_pass.cc:72] Find all_reduce operators: 401. To make the speed faster, some all_reduce ops are fused during training, after fusion, the number of all_reduce ops is 255.
+I1029 10:44:03.584228 32687 build_strategy.cc:363] SeqOnlyAllReduceOps:0, num_trainers:1
+I1029 10:44:39.690382 32687 parallel_executor.cc:285] Inplace strategy is enabled, when build_strategy.enable_inplace = True
+I1029 10:44:42.244774 32687 parallel_executor.cc:368] Garbage collection strategy is enabled, when FLAGS_eager_delete_tensor_gb = 0
+W1029 10:48:20.253201 32687 init.cc:212] *** Aborted at 1572317300 (unix time) try "date -d @1572317300" if you are using GNU date ***
+W1029 10:48:20.255347 32687 init.cc:212] PC: @                0x0 (unknown)
+W1029 10:48:20.255458 32687 init.cc:212] *** SIGTERM (@0x1f80000785a) received by PID 32687 (TID 0x7f0f71d25700) from PID 30810; stack trace: ***
+W1029 10:48:20.257107 32687 init.cc:212]     @     0x7f0f714ef160 (unknown)
+W1029 10:48:20.258708 32687 init.cc:212]     @     0x7f0f714eb3cc __pthread_cond_wait
+W1029 10:48:20.259734 32687 init.cc:212]     @     0x7f0f249d33cc std::condition_variable::wait()
+W1029 10:48:20.263964 32687 init.cc:212]     @     0x7f0f008e990d paddle::framework::details::FastThreadedSSAGraphExecutor::Run()
+W1029 10:48:20.265229 32687 init.cc:212]     @     0x7f0f0084a6a7 _ZNSt17_Function_handlerIFvvEZN6paddle9framework7details29ScopeBufferedSSAGraphExecutor3RunERKSt6vectorISsSaISsEEEUlvE_E9_M_invokeERKSt9_Any_data
+W1029 10:48:20.268503 32687 init.cc:212]     @     0x7f0f0084f4bf paddle::framework::details::ScopeBufferedMonitor::Apply()
+W1029 10:48:20.270135 32687 init.cc:212]     @     0x7f0f0084ae86 paddle::framework::details::ScopeBufferedSSAGraphExecutor::Run()
+W1029 10:48:20.272866 32687 init.cc:212]     @     0x7f0efe5ed038 paddle::framework::ParallelExecutor::Run()
+W1029 10:48:20.273551 32687 init.cc:212]     @     0x7f0efe3d0e78 _ZZN8pybind1112cpp_function10initializeIZN6paddle6pybindL22pybind11_init_core_avxERNS_6moduleEEUlRNS2_9framework16ParallelExecutorERKSt6vectorISsSaISsEEE188_S9_INS6_9LoDTensorESaISF_EEIS8_SD_EINS_4nameENS_9is_methodENS_7siblingEEEEvOT_PFT0_DpT1_EDpRKT2_ENUlRNS_6detail13function_callEE1_4_FUNESY_
+W1029 10:48:20.274988 32687 init.cc:212]     @     0x7f0efe41af56 pybind11::cpp_function::dispatcher()
+W1029 10:48:20.276706 32687 init.cc:212]     @     0x7f0f71808cc8 PyEval_EvalFrameEx
+W1029 10:48:20.278395 32687 init.cc:212]     @     0x7f0f7180b35d PyEval_EvalCodeEx
+W1029 10:48:20.280076 32687 init.cc:212]     @     0x7f0f71808d50 PyEval_EvalFrameEx
+W1029 10:48:20.281765 32687 init.cc:212]     @     0x7f0f7180b35d PyEval_EvalCodeEx
+W1029 10:48:20.283442 32687 init.cc:212]     @     0x7f0f71808d50 PyEval_EvalFrameEx
+W1029 10:48:20.285133 32687 init.cc:212]     @     0x7f0f7180b35d PyEval_EvalCodeEx
+W1029 10:48:20.286808 32687 init.cc:212]     @     0x7f0f71808d50 PyEval_EvalFrameEx
+W1029 10:48:20.288502 32687 init.cc:212]     @     0x7f0f7180b35d PyEval_EvalCodeEx
+W1029 10:48:20.290176 32687 init.cc:212]     @     0x7f0f71808d50 PyEval_EvalFrameEx
+W1029 10:48:20.291870 32687 init.cc:212]     @     0x7f0f7180b35d PyEval_EvalCodeEx
+W1029 10:48:20.293542 32687 init.cc:212]     @     0x7f0f7180b492 PyEval_EvalCode
+W1029 10:48:20.295228 32687 init.cc:212]     @     0x7f0f718351a2 PyRun_FileExFlags
+W1029 10:48:20.296922 32687 init.cc:212]     @     0x7f0f71836539 PyRun_SimpleFileExFlags
+W1029 10:48:20.298590 32687 init.cc:212]     @     0x7f0f7184c1bd Py_Main
+W1029 10:48:20.300307 32687 init.cc:212]     @     0x7f0f70a49bd5 __libc_start_main
+W1029 10:48:20.300364 32687 init.cc:212]     @           0x4007a1 (unknown)
+W1029 10:48:20.302006 32687 init.cc:212]     @                0x0 (unknown)

paddlepalm/backbone/utils/transformer.py

@@ -23,6 +23,35 @@ from functools import partial
 import paddle.fluid as fluid
 import paddle.fluid.layers as layers
 
+from paddle.fluid.layer_helper import LayerHelper as LayerHelper
+def layer_norm(x, begin_norm_axis=1, epsilon=1e-6, param_attr=None, bias_attr=None):
+    helper = LayerHelper('layer_norm', **locals())
+    mean = layers.reduce_mean(x, dim=begin_norm_axis, keep_dim=True)
+    shift_x = layers.elementwise_sub(x=x, y=mean, axis=0)
+    variance = layers.reduce_mean(layers.square(shift_x), dim=begin_norm_axis, keep_dim=True)
+    r_stdev = layers.rsqrt(variance + epsilon)
+    norm_x = layers.elementwise_mul(x=shift_x, y=r_stdev, axis=0)
+
+    param_shape = [reduce(lambda x, y: x * y, norm_x.shape[begin_norm_axis:])]
+    param_dtype = norm_x.dtype
+    scale = helper.create_parameter(
+        attr=param_attr,
+        shape=param_shape,
+        dtype=param_dtype,
+        default_initializer=fluid.initializer.Constant(1.))
+    bias = helper.create_parameter(
+        attr=bias_attr,
+        shape=param_shape,
+        dtype=param_dtype,
+        is_bias=True,
+        default_initializer=fluid.initializer.Constant(0.))
+
+    out = layers.elementwise_mul(x=norm_x, y=scale, axis=-1)
+    out = layers.elementwise_add(x=out, y=bias, axis=-1)
+
+    return out
+
+
 def multi_head_attention(queries,
                          keys,
                          values,
@@ -209,7 +238,7 @@ def pre_post_process_layer(prev_out, out, process_cmd, dropout_rate=0.,
             out_dtype = out.dtype
             if out_dtype == fluid.core.VarDesc.VarType.FP16:
                 out = layers.cast(x=out, dtype="float32")
-            out = layers.layer_norm(
+            out = layer_norm(
                 out,
                 begin_norm_axis=len(out.shape) - 1,
                 param_attr=fluid.ParamAttr(

paddlepalm/reader/match4ernie.py → paddlepalm/reader/cls.py

@@ -36,6 +36,8 @@ class Reader(reader):
 
         self._batch_size = config['batch_size']
         self._max_seq_len = config['max_seq_len']
+        self._num_classes = config['n_classes']
+
         if phase == 'train':
             self._input_file = config['train_file']
             self._num_epochs = None # 防止iteartor终止
@@ -91,6 +93,7 @@ class Reader(reader):
             return outputs
 
         for batch in self._data_generator():
+            print(batch)
             yield list_to_dict(batch)
 
     def get_epoch_outputs(self):

paddlepalm/reader/mlm.py

@@ -15,6 +15,7 @@
 
 from paddlepalm.interface import reader
 from paddlepalm.reader.utils.reader4ernie import MaskLMReader
+import numpy as np
 
 class Reader(reader):
     
@@ -81,6 +82,8 @@ class Reader(reader):
             return outputs
 
         for batch in self._data_generator():
+            # print(np.shape(list_to_dict(batch)['token_ids']))
+            # print(list_to_dict(batch)['mask_label'].tolist())
             yield list_to_dict(batch)
 
     def get_epoch_outputs(self):

paddlepalm/task_paradigm/cls.py

@@ -21,15 +21,29 @@ class TaskParadigm(task_paradigm):
     '''
     classification
     '''
-    def __init___(self, config, phase):
+    def __init___(self, config, phase, backbone_config=None):
         self._is_training = phase == 'train'
-        self.sent_emb_size = config['hidden_size']
+        self._hidden_size = backbone_config['hidden_size']
         self.num_classes = config['n_classes']
     
+        if 'initializer_range' in config:
+            self._param_initializer = config['initializer_range']
+        else:
+            self._param_initializer = fluid.initializer.TruncatedNormal(
+                scale=backbone_config.get('initializer_range', 0.02))
+        if 'dropout_prob' in config:
+            self._dropout_prob = config['dropout_prob']
+        else:
+            self._dropout_prob = backbone_config.get('hidden_dropout_prob', 0.0)
+
     @property
     def inputs_attrs(self):
-        return {'bakcbone': {"sentence_emb": [-1, self.sent_emb_size], 'float32']},
-                'reader': {"label_ids": [[-1, 1], 'int64']}}
+        if self._is_training:
+            reader = {"label_ids": [[-1, 1], 'int64']}
+        else:
+            reader = {}
+        bb = {"sentence_embedding": [[-1, self._hidden_size], 'float32']}
+        return {'reader': reader, 'backbone': bb}
 
     @property
     def outputs_attrs(self):
@@ -39,22 +53,29 @@ class TaskParadigm(task_paradigm):
             return {'logits': [-1, self.num_classes], 'float32'}
 
     def build(self, **inputs):
-        sent_emb = inputs['backbone']['sentence_emb']
+        sent_emb = inputs['backbone']['sentence_embedding']
         label_ids = inputs['reader']['label_ids']
 
+        if self._is_training:
+            cls_feats = fluid.layers.dropout(
+                x=sent_emb,
+                dropout_prob=self._dropout_prob,
+                dropout_implementation="upscale_in_train")
+
         logits = fluid.layers.fc(
-            input=ent_emb
+            input=sent_emb,
             size=self.num_classes,
             param_attr=fluid.ParamAttr(
                 name="cls_out_w",
-                initializer=fluid.initializer.TruncatedNormal(scale=0.1)),
+                initializer=self._param_initializer),
             bias_attr=fluid.ParamAttr(
                 name="cls_out_b", initializer=fluid.initializer.Constant(0.)))
 
-        loss = fluid.layers.softmax_with_cross_entropy(
-            logits=logits, label=label_ids)
-        loss = layers.mean(loss)
         if self._is_training:
+            loss = fluid.layers.softmax_with_cross_entropy(
+                logits=logits, label=label_ids)
+            loss = layers.mean(loss)
             return {"loss": loss}
         else:
             return {"logits":logits}
+

paddlepalm/task_paradigm/match.py

@@ -24,6 +24,17 @@ class TaskParadigm(task_paradigm):
     def __init__(self, config, phase, backbone_config=None):
         self._is_training = phase == 'train'
         self._hidden_size = backbone_config['hidden_size']
+
+        if 'initializer_range' in config:
+            self._param_initializer = config['initializer_range']
+        else:
+            self._param_initializer = fluid.initializer.TruncatedNormal(
+                scale=backbone_config.get('initializer_range', 0.02))
+        if 'dropout_prob' in config:
+            self._dropout_prob = config['dropout_prob']
+        else:
+            self._dropout_prob = backbone_config.get('hidden_dropout_prob', 0.0)
+
     
     @property
     def inputs_attrs(self):
@@ -46,16 +57,18 @@ class TaskParadigm(task_paradigm):
             labels = inputs["reader"]["label_ids"] 
         cls_feats = inputs["backbone"]["sentence_pair_embedding"]
 
-        cls_feats = fluid.layers.dropout(
-            x=cls_feats,
-            dropout_prob=0.1,
-            dropout_implementation="upscale_in_train")
+        if self._is_training:
+            cls_feats = fluid.layers.dropout(
+                x=cls_feats,
+                dropout_prob=self._dropout_prob,
+                dropout_implementation="upscale_in_train")
+
         logits = fluid.layers.fc(
             input=cls_feats,
             size=2,
             param_attr=fluid.ParamAttr(
                 name="cls_out_w",
-                initializer=fluid.initializer.TruncatedNormal(scale=0.02)),
+                initializer=self._param_initializer),
             bias_attr=fluid.ParamAttr(
                 name="cls_out_b",
                 initializer=fluid.initializer.Constant(0.)))

run_demo2.sh

-export CUDA_VISIBLE_DEVICES=0
-
-python demo2.py
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+while true
+do
+    python demo2.py
+done
 

run_demo3.sh

+export CUDA_VISIBLE_DEVICES=0,1
+
+python demo3.py
+