refactor demo

config_demo1.yaml → demo/demo1/config.yaml

@@ -3,7 +3,7 @@ task_instance: "mrqa"
 save_path: "output_model/firstrun"
 
 backbone: "bert"
-backbone_config_path: "pretrain_model/bert/bert_config.json"
+backbone_config_path: "../../pretrain_model/bert/bert_config.json"
 
 batch_size: 4
 num_epochs: 2

demo1_tasks/mrqa.yaml → demo/demo1/mrqa.yaml

@@ -2,7 +2,7 @@ train_file: data/mrqa/train.json
 reader: mrc
 paradigm: mrc
 
-vocab_path: "pretrain_model/bert/vocab.txt"
+vocab_path: "../../pretrain_model/bert/vocab.txt"
 do_lower_case: True
 max_seq_len: 512
 doc_stride: 128

demo1.py → demo/demo1/run.py

 import paddlepalm as palm
 
 if __name__ == '__main__':
-    controller = palm.Controller('config_demo1.yaml', task_dir='demo1_tasks')
-    controller.load_pretrain('pretrain_model/bert/params')
+    controller = palm.Controller('config.yaml')
+    controller.load_pretrain('../../pretrain_model/bert/params')
     controller.train()
 

run_demo1.sh → demo/demo1/run.sh

 export CUDA_VISIBLE_DEVICES=0
 
-python demo1.py
+python run.py
 

config_demo2.yaml → demo/demo2/config.yaml

@@ -5,9 +5,9 @@ mix_ratio: 1.0, 0.5, 0.5
 save_path: "output_model/secondrun"
 
 backbone: "ernie"
-backbone_config_path: "pretrain_model/ernie/ernie_config.json"
+backbone_config_path: "../../pretrain_model/ernie/ernie_config.json"
 
-vocab_path: "pretrain_model/ernie/vocab.txt"
+vocab_path: "../../pretrain_model/ernie/vocab.txt"
 do_lower_case: True
 max_seq_len: 512
 

demo/demo2/data/mrqa/convert.py

+# coding: utf-8
+f='mrqa-combined.train.raw.json'
+import json
+a=json.load(open(f))
+a=a['data']
+writer = open('train.json','w')
+    
+for s in a:
+    p = s['paragraphs']
+    assert len(p) == 1
+    p = p[0]
+    q = {}
+    q['context'] = p['context']
+    q['qa_list'] = p['qas']
+    writer.write(json.dumps(q)+'\n')
+    
+writer.close()
+    

demo2.py → demo/demo2/run.py

 import paddlepalm as palm
 
 if __name__ == '__main__':
-    controller = palm.Controller('config_demo2.yaml', task_dir='demo2_tasks')
-    controller.load_pretrain('pretrain_model/ernie/params')
+    controller = palm.Controller('config.yaml', task_dir='tasks')
+    controller.load_pretrain('../../pretrain_model/ernie/params')
     controller.train()
 
-    controller = palm.Controller(config='config_demo2.yaml', task_dir='demo2_tasks', for_train=False)
+    controller = palm.Controller(config='config.yaml', task_dir='tasks', for_train=False)
     controller.pred('mrqa', inference_model_dir='output_model/secondrun/mrqa/infer_model')
 
 

demo/demo2/run.sh

+export CUDA_VISIBLE_DEVICES=0,1
+python run.py 
+

config_demo3.yaml → demo/demo3/config.yaml

@@ -5,9 +5,9 @@ task_reuse_tag: 0,0,1,1,0,2
 save_path: "output_model/thirdrun"
 
 backbone: "ernie"
-backbone_config_path: "pretrain_model/ernie/ernie_config.json"
+backbone_config_path: "../../pretrain_model/ernie/ernie_config.json"
 
-vocab_path: "pretrain_model/ernie/vocab.txt"
+vocab_path: "../../pretrain_model/ernie/vocab.txt"
 do_lower_case: True
 max_seq_len: 512
 

demo3.py → demo/demo3/run.py

 import paddlepalm as palm
 
 if __name__ == '__main__':
-    controller = palm.Controller('config_demo3.yaml', task_dir='demo3_tasks')
-    controller.load_pretrain('pretrain_model/ernie/params')
+    controller = palm.Controller('config.yaml', task_dir='tasks')
+    controller.load_pretrain('../../pretrain_model/ernie/params')
     controller.train()
 

run_demo3.sh → demo/demo3/run.sh

 export CUDA_VISIBLE_DEVICES=0
 
-python demo3.py
+python run.py
 

run_demo2.sh

-export CUDA_VISIBLE_DEVICES=0
-python demo2.py 
-