fix bugs

demo.py → demo1.py

 import paddlepalm as palm
 
 if __name__ == '__main__':
-    controller = palm.Controller('config.yaml', task_dir='task_instance')
+    controller = palm.Controller('demo1_config.yaml', task_dir='demo1_tasks')
     controller.load_pretrain('pretrain_model/ernie/params')
     controller.train()
 
-    controller = palm.Controller(config='config.yaml', task_dir='task_instance', for_train=False)
+    controller = palm.Controller(config='demo1_config.yaml', task_dir='demo1_tasks', for_train=False)
     controller.pred('mrqa', inference_model_dir='output_model/firstrun/infer_model')
 

demo1_config.yaml

+task_instance: "mrqa"
+target_tag: 1
+mix_ratio: 1.0
+
+save_path: "output_model/firstrun"
+
+backbone: "ernie"
+backbone_config_path: "pretrain_model/ernie/ernie_config.json"
+
+vocab_path: "pretrain_model/ernie/vocab.txt"
+do_lower_case: True
+max_seq_len: 512
+
+batch_size: 5
+num_epochs: 2
+optimizer: "adam"
+learning_rate: 3e-5
+warmup_proportion: 0.1
+weight_decay: 0.1
+

demo2.py

+import paddlepalm as palm
+
+if __name__ == '__main__':
+    controller = palm.Controller('demo2_config.yaml', task_dir='demo2_tasks')
+    controller.load_pretrain('pretrain_model/ernie/params')
+    controller.train()
+
+    controller = palm.Controller(config='demo2_config.yaml', task_dir='demo2_tasks', for_train=False)
+    controller.pred('mrqa', inference_model_dir='output_model/secondrun/infer_model')
+

config.yaml → demo2_config.yaml

@@ -2,7 +2,7 @@ task_instance: "mrqa, match4mrqa"
 target_tag: 1, 0
 mix_ratio: 1.0, 0.5
 
-save_path: "output_model/firstrun"
+save_path: "output_model/secondrun"
 
 backbone: "ernie"
 backbone_config_path: "pretrain_model/ernie/ernie_config.json"

demo2_tasks/mrqa.yaml

+train_file: data/mrqa/mrqa-combined.train.raw.json
+pred_file: data/mrqa/mrqa-combined.dev.raw.json
+pred_output_path: 'mrqa_output'
+reader: mrc4ernie
+paradigm: mrc
+doc_stride: 128
+max_query_len: 64
+max_answer_len: 30
+n_best_size: 20
+null_score_diff_threshold: 0.0
+verbose: False

mrqa_output/predictions.json

+{
+    "3f02f171c82e49828580007a71eefc31": "Ethan", 
+    "98d0b8ce19d1434abdb42aa01e83db61": "MonetDB.R   [SEP]   I   am   trying   to   import   a   dataset   of   217000   records   (Jeopardy", 
+    "f0bc45a4dd7a4d8abf91a5e4fb25fe57": "James'   epitaph   contains   this   line:   Murdered   by   a", 
+    "2dc90736586049d298a10ed93567f0db": "Steve   Tyler   and   Joe   Perry   at   2010   \n Aerosmith   concert.   [SEP]   [SEP]   The   Yardbirds", 
+    "9aa1a16d4d1c4d8c874dc8cad32d2c49": "the   Beyond.   [SEP]   [SEP]   Authors     Me   and   My   Kindle   [SEP]   But   if   you   read   Irving", 
+    "4113be8423d14a4790a5c5e569d4595a": "Bartholomew", 
+    "4b7a7b560b094e309c10e8dd11e9c6fc": "Am   &   Qantas   in   the   late   '70s,   it   was   \n basically   a   roped-off   part   of   the   economy   cabin   with   free   drinks.   business   class.", 
+    "1426088cb6494263906e1111064d5d72": "a   nova   srie   nasceu   na   cabea   de   Barbra   Streisan", 
+    "3e24b66b256a4e28ae00280d8960803a": "Jim", 
+    "fda746982dd848b89c40b3fc8a8e56d2": "Jackie   Glea", 
+    "aa0179e972f94530a8125180d25aa3b4": "Cows   Eat   Grass", 
+    "b5da96ef279448aca61b049042467a4a": "periods", 
+    "cc15c92b096a4be898611ed8d3869b4e": "Galileo   \n was   under   house   arrest   for   espousing   this   man's", 
+    "43ee9fcf44a148348923d4d3f22d7d98": "Bulova", 
+    "8824fe46b8b442699c7b8ef0b25ef995": "Sheen", 
+    "e04ef70e4d034c2e9d56dc96d662d9cb": "her   tragic   death.   ...   prime   \n minister   Tony   Blair   dubbed   Diana", 
+    "b354d2f970eb457bb5a305d94ba1a777": "John   Spencer   Of   'West   Wing", 
+    "40b52c7c4f6b46ecb65a0541a69e92a6": "Langham   of   Veronica's...   [SEP]   [SEP]   Daydream   Believers:   The   Monkees", 
+    "10abdca0abb04882a56e94745a587f28": "John   Adams", 
+    "4a360242cbd7421aab8ab86a30b9766b": "of   revolutionary   scientific   works.   The   Origin   of", 
+    "670dc24a0fb04455bbb889733bf002a1": "Hume", 
+    "9f294e3878034f48bd421ed0eb96f2a3": "Mount   Rainier", 
+    "beb0368980824527ac58e7f983bca97f": "it   is   to   merge   with   KLM,   the   Dutch   airline.   ...   The   \n takeover   will   create   Europe's   largest   airline,", 
+    "90c42d76f3dd44cf86ac3d5b945acee1": "Cleopatra", 
+    "45c25b136e9947309d3f3199a5eac397": "Appian   Way   -   Wikipedia   [SEP]   The   Appian", 
+    "2a26ba2446794ed38d4633091a001a4d": "s", 
+    "4bce3bce06ad4b0d85022ac9e7d25bed": "Airlines   on   flights   between   \n Newark   Liberty", 
+    "483db64145af4cf2955ec689c62d2ab5": "Vain"
+}

paddlepalm/mtl_controller.py

@@ -422,7 +422,7 @@ class Controller(object):
             prefixes.append(inst.name)
             mrs.append(inst.mix_ratio)
 
-        joint_iterator_fn = create_joint_iterator_fn(iterators, prefixes, joint_shape_and_dtypes, mrs, name_to_position, dev_count=dev_count, verbose=VERBOSE)
+        joint_iterator_fn = create_joint_iterator_fn(iterators, prefixes, joint_shape_and_dtypes, mrs, name_to_position, dev_count=dev_count, verbose=VERBOSE, batch_size=main_conf['batch_size'])
 
         input_attrs = [[i, j, k] for i, (j,k) in zip(joint_input_names, joint_shape_and_dtypes)]
         pred_input_attrs = [[i, j, k] for i, (j,k) in zip(pred_joint_input_names, pred_joint_shape_and_dtypes)]
@@ -488,10 +488,9 @@ class Controller(object):
 
         bb_fetches = {k: v.name for k,v in bb_output_vars.items()}
         task_fetches = {k: v.name for k,v in task_output_vars.items()}
-        old = len(bb_fetches)+len(task_fetches) # for debug
-        fetches = bb_fetches.copy()
-        fetches.update(task_fetches)
-        assert len(fetches) == old # for debug
+        # fetches = bb_fetches.copy() # 注意！框架在多卡时无法fetch变长维度的tensor，这里加入bb的out后会挂
+        # fetches.update(task_fetches)
+        fetches = task_fetches
         fetches['__task_id'] = net_inputs['__task_id'].name
 
         # compute loss
@@ -505,7 +504,8 @@ class Controller(object):
         num_examples = main_reader.num_examples
         for inst in instances:
             max_train_steps = int(main_conf['num_epochs']* inst.mix_ratio * num_examples) // main_conf['batch_size']  // dev_count
-            print('{}: expected train steps {}.'.format(inst.name, max_train_steps))
+            if inst.is_target:
+                print('{}: expected train steps {}.'.format(inst.name, max_train_steps))
             inst.steps_pur_epoch = inst.reader['train'].num_examples // main_conf['batch_size']  // dev_count
             inst.expected_train_steps = max_train_steps
 
@@ -622,12 +622,11 @@ class Controller(object):
         epoch = 0
         time_begin = time.time()
         backbone_buffer = []
-        task_buffer = [[]] * num_instances
         while not train_finish():
             rt_outputs = self.exe.run(train_program, fetch_list=fetch_list)
             rt_outputs = {k:v for k,v in zip(fetch_names, rt_outputs)}
             rt_task_id = np.squeeze(rt_outputs['__task_id']).tolist()
-            assert (not isinstance(rt_task_id, list)) or len(set(rt_task_id)) == 1
+            assert (not isinstance(rt_task_id, list)) or len(set(rt_task_id)) == 1, rt_task_id
             rt_task_id = rt_task_id[0] if isinstance(rt_task_id, list) else rt_task_id
             cur_task = instances[rt_task_id]
 
@@ -635,8 +634,7 @@ class Controller(object):
             backbone_buffer.append(backbone.postprocess(backbone_rt_outputs))
             
             task_rt_outputs = {k[len(cur_task.name+'/'):]: v for k,v in rt_outputs.items() if k.startswith(cur_task.name+'/')}
-            temp = instances[rt_task_id].task_layer['train'].postprocess(task_rt_outputs)
-            task_buffer[rt_task_id].append(temp)
+            instances[rt_task_id].task_layer['train'].postprocess(task_rt_outputs)
 
             global_step += 1
             # if cur_task.is_target:

paddlepalm/reader/mrc4ernie.py

@@ -30,6 +30,7 @@ class Reader(reader):
             max_seq_len=config['max_seq_len'],
             do_lower_case=config.get('do_lower_case', False),
             tokenizer='FullTokenizer',
+            for_cn=config.get('for_cn', False),
             doc_stride=config['doc_stride'],
             max_query_length=config['max_query_len'],
             random_seed=config.get('seed', None))

paddlepalm/task_paradigm/match.py

@@ -42,7 +42,8 @@ class TaskParadigm(task_paradigm):
             return {"logits": [[-1, 1], 'float32']}
 
     def build(self, inputs):
-        labels = inputs["reader"]["label_ids"] 
+        if self._is_training:
+            labels = inputs["reader"]["label_ids"] 
         cls_feats = inputs["backbone"]["sentence_pair_embedding"]
 
         cls_feats = fluid.layers.dropout(
@@ -58,11 +59,11 @@ class TaskParadigm(task_paradigm):
             bias_attr=fluid.ParamAttr(
                 name="cls_out_b",
                 initializer=fluid.initializer.Constant(0.)))
-        ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
-            logits=logits, label=labels, return_softmax=True)
-        loss = fluid.layers.mean(x=ce_loss)
 
         if self._is_training:
+            ce_loss, probs = fluid.layers.softmax_with_cross_entropy(
+                logits=logits, label=labels, return_softmax=True)
+            loss = fluid.layers.mean(x=ce_loss)
             return {'loss': loss}
         else:
             return {'logits': logits}

paddlepalm/task_paradigm/mrc.py

@@ -65,9 +65,7 @@ class TaskParadigm(task_paradigm):
     @property
     def outputs_attr(self):
         if self._is_training:
-            return {'start_logits': [[-1, -1, 1], 'float32'],
-                    'end_logits': [[-1, -1, 1], 'float32'],
-                    'loss': [[1], 'float32']}
+            return {'loss': [[1], 'float32']}
         else:
             return {'start_logits': [[-1, -1, 1], 'float32'],
                     'end_logits': [[-1, -1, 1], 'float32'],
@@ -106,16 +104,14 @@ class TaskParadigm(task_paradigm):
             start_loss = _compute_single_loss(start_logits, start_positions)
             end_loss = _compute_single_loss(end_logits, end_positions)
             total_loss = (start_loss + end_loss) / 2.0
-            return {'start_logits': start_logits,
-                    'end_logits': end_logits,
-                    'loss': total_loss}
+            return {'loss': total_loss}
         else:
             return {'start_logits': start_logits,
                     'end_logits': end_logits,
                     'unique_ids': unique_id}
 
 
-    def postprocess(self, rt_outputs):                                                                                                                                                   
+    def postprocess(self, rt_outputs):
         """this func will be called after each step(batch) of training/evaluating/predicting process."""
         if not self._is_training:
             unique_ids = np.squeeze(rt_outputs['unique_ids'], -1)

paddlepalm/utils/reader_helper.py

@@ -48,6 +48,20 @@ def _zero_batch(attrs):
     return [np.zeros(shape=shape, dtype=dtype) for shape, dtype in pos_attrs]
 
 
+def _zero_batch_x(attrs, batch_size):
+    pos_attrs = []
+    for shape, dtype in attrs:
+        # pos_shape = [size if size and size > 0 else 5 for size in shape]
+        pos_shape = [size for size in shape]
+        if pos_shape[0] == -1:
+            pos_shape[0] = batch_size
+        if pos_shape[1] == -1:
+            pos_shape[1] = 512 # max seq len
+        pos_attrs.append([pos_shape, dtype])
+
+    return [np.zeros(shape=shape, dtype=dtype) for shape, dtype in pos_attrs]
+
+
 def create_net_inputs(input_attrs, async=False, iterator_fn=None, dev_count=1, n_prefetch=1):
     inputs = []
     ret = {}
@@ -92,10 +106,11 @@ def create_iterator_fn(iterator, iterator_prefix, shape_and_dtypes, outname_to_p
     return iterator
 
 
-def create_joint_iterator_fn(iterators, iterator_prefixes, joint_shape_and_dtypes, mrs, outname_to_pos, dev_count=1, keep_one_task=True, verbose=0):
+def create_joint_iterator_fn(iterators, iterator_prefixes, joint_shape_and_dtypes, mrs, outname_to_pos, dev_count=1, keep_one_task=True, verbose=0, batch_size=None):
     """
         joint_shape_and_dtypes: 本质上是根据bb和parad的attr设定的，并且由reader中的attr自动填充-1（可变）维度得到，因此通过与iterator的校验可以完成runtime的batch正确性检查
     """
+
     task_ids = range(len(iterators))
     weights = [mr / float(sum(mrs)) for mr in mrs]
     if not keep_one_task:
@@ -129,7 +144,6 @@ def create_joint_iterator_fn(iterators, iterator_prefixes, joint_shape_and_dtype
         v = verbose
         while True:
             id = np.random.choice(task_ids, p=weights)
-            # results = _zero_batch(joint_shape_and_dtypes)
             results = fake_batch
             if v > 0:
                 print('----- debug joint iterator -----')
@@ -138,6 +152,8 @@ def create_joint_iterator_fn(iterators, iterator_prefixes, joint_shape_and_dtype
             results[0] = task_id_tensor
             
             for i in range(dev_count):
+                # results = _zero_batch(joint_shape_and_dtypes, batch_size=batch_size)
+                # results[0] = task_id_tensor
                 if id in outbuf:
                     outputs = outbuf[id]
                     del outbuf[id]

run_demo.sh → run_demo1.sh

-export CUDA_VISIBLE_DEVICES=0
+export CUDA_VISIBLE_DEVICES=0,1,2,3
 export FLAGS_fraction_of_gpu_memory_to_use=0.1
 export FLAGS_eager_delete_tensor_gb=0
 
-python demo.py
+python demo1.py
 

run_demo2.sh

+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export FLAGS_fraction_of_gpu_memory_to_use=0.1
+export FLAGS_eager_delete_tensor_gb=0
+
+python demo2.py
+

task_instance/mlm4mrqa.yaml

-train_file: "data/mlm4mrqa"
-mix_ratio: 0.4
-batch_size: 4
-in_tokens: False
-generate_neg_sample: False