merge api-r0.3

download_models.py

+# -*- coding: UTF-8 -*-
+#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import paddlepalm as palm
+import sys
+import argparse
+ 
+# create parser
+parser = argparse.ArgumentParser(prog='download_models.py', usage='python %(prog)s -l | -d <model_name> [-h]\n\nFor example,\n\tpython %(prog)s -d bert-en-uncased-large ',description = 'Download pretrain models for initializing params of backbones. ')
+parser1= parser.add_argument_group("required arguments")
+parser1.add_argument('-l','--list', action = 'store_true', help = 'show the list of available pretrain models', default = False)
+parser1.add_argument('-d','--download', action = 'store', help = 'download pretrain models. The available pretrain models can be listed by run "python download_models.py -l"') 
+args = parser.parse_args()
+
+if(args.list):
+  palm.downloader.ls('pretrain')
+elif(args.download):
+  print('download~~~')
+  print(args.download)
+  palm.downloader.download('pretrain', args.download)
+else:
+  print (parser.parse_args(['-h']))

paddlepalm/_downloader.py

@@ -137,8 +137,8 @@ def download(item, scope='all', path='.'):
     """download an item. The available scopes and contained items can be showed with `paddlepalm.downloader.ls`.
 
     Args:
-        scope: the scope the item belongs to.
         item: the item to download.
+        scope: the scope of the item to download.
         path: the target dir to download to. Default is `.`, means current dir.
     """
     item = item.lower()
@@ -180,4 +180,5 @@ def ls(item='all', scope='all'):
             print ('Available {} items: '.format(i))
             _ls(i, scope, l)
 
+
     

paddlepalm/distribute/reader.py

@@ -56,6 +56,7 @@ def yield_pieces(data, distribute_strategy, batch_size):
             # print(len(temp))
             yield temp
 
+
 def data_feeder(reader, postprocess_fn=None, prefetch_steps=2):
     if postprocess_fn is None:
         def postprocess_fn(batch):
@@ -101,20 +102,19 @@ def data_feeder(reader, postprocess_fn=None, prefetch_steps=2):
                 batch_buf.append(batch)
                 flag_buf.append(flag)
             yield batch_buf, flag_buf
-        else: 
+        else:
             break
     queue.join()
 
 
 
-
 def decode_fake(nums, mask, bs):
     n_t = 0
-    for flag in mask: 
+    for flag in mask:
         if not flag:
             break
         n_t = n_t + 1
-    
+
     n_f = len(mask) - n_t
     p1 = nums - (n_t-1) * bs
     each_f = p1 / (n_f+1)

paddlepalm/head/cls.py

@@ -38,7 +38,6 @@ class Classify(Head):
         self._param_initializer = fluid.initializer.TruncatedNormal(
             scale=param_initializer_range)
         self._preds = []
-        self._probs = []
 
     @property
     def inputs_attrs(self):
@@ -53,8 +52,7 @@ class Classify(Head):
         if self._is_training:
             return {'loss': [[1], 'float32']}
         else:
-            return {'logits': [[-1, self.num_classes], 'float32'],
-                    'probs': [[-1, self.num_classes], 'float32']}
+            return {'logits': [[-1, self.num_classes], 'float32']}
 
     def build(self, inputs, scope_name=''):
         sent_emb = inputs['backbone']['sentence_embedding']
@@ -74,10 +72,10 @@ class Classify(Head):
             bias_attr=fluid.ParamAttr(
                 name=scope_name+"cls_out_b", initializer=fluid.initializer.Constant(0.)))
 
-        probs = fluid.layers.softmax(logits)
         if self._is_training:
+            inputs = fluid.layers.softmax(logits)
             loss = fluid.layers.cross_entropy(
-                input=probs, label=label_ids)
+                input=inputs, label=label_ids)
             loss = layers.mean(loss)
             return {"loss": loss}
         else:

paddlepalm/lr_sched/slanted_triangular_schedualer.py

 
-from paddlepalm.lr_sched.base_schedualer import Schedualer
+from paddlepalm.lr_sched.schedualer import Schedualer
 from paddle import fluid
 
 class TriangularSchedualer(Schedualer):
@@ -14,7 +14,7 @@ class TriangularSchedualer(Schedualer):
             num_train_steps: the number of train steps.
 
         """
-        Schedualer.__init__(self)
+        BaseSchedualer.__init__(self)
         assert num_train_steps > warmup_steps > 0
         self.warmup_steps = warmup_steps
         self.num_train_steps = num_train_steps

paddlepalm/lr_sched/warmup_schedualer.py

 
-from paddlepalm.lr_sched.base_schedualer import Schedualer
+from paddlepalm.lr_sched.schedualer import Schedualer
 import paddle.fluid as fluid
 
 def WarmupSchedualer(Schedualer):

test/test3/run.py

@@ -4,16 +4,16 @@ import json
 
 if __name__ == '__main__':
 
-    max_seqlen = 256
-    batch_size = 8
-    num_epochs = 10
-    lr = 5e-5
-    vocab_path = './pretrain/ernie-ch-uncased-base/vocab.txt'
-
-    train_file = './data/chnsenticorp/train.tsv'
-    predict_file = './data/chnsenticorp/test.tsv'
-    random_seed = 1
-    config = json.load(open('./pretrain/ernie-ch-uncased-base/ernie_config.json'))
+    max_seqlen = 512
+    batch_size = 4
+    num_epochs = 2
+    lr = 1e-3
+    vocab_path = './pretrain/ernie/vocab.txt'
+
+    train_file = './data/cls4mrqa/train.tsv'
+    predict_file = './data/cls4mrqa/dev.tsv'
+
+    config = json.load(open('./pretrain/ernie/ernie_config.json'))
     # ernie = palm.backbone.ERNIE(...)
     ernie = palm.backbone.ERNIE.from_config(config)
 
@@ -23,59 +23,132 @@ if __name__ == '__main__':
     # subj_trainer = palm.Trainer('subj_cls', cls_reader3, cls)
 
     # 创建该分类任务的reader，由诸多参数控制数据集读入格式、文件数量、预处理规则等
-    cls_reader = palm.reader.ClassifyReader(vocab_path, max_seqlen, seed=random_seed)
-    predict_cls_reader = palm.reader.ClassifyReader(vocab_path, max_seqlen, seed=random_seed, phase='predict')
+    cls_reader = palm.reader.ClassifyReader(vocab_path, max_seqlen)
+<<<<<<< HEAD:test/test2/run.py
+    cls_reader2 = palm.reader.ClassifyReader(vocab_path, max_seqlen)
+=======
+    predict_cls_reader = palm.reader.ClassifyReader(vocab_path, max_seqlen, phase='predict')
+>>>>>>> remotes/upstream/r0.3-api:test/test3/run.py
+    print(cls_reader.outputs_attr)
+    print(predict_cls_reader.outputs_attr)
     # 不同的backbone会对任务reader有不同的特征要求，例如对于分类任务，基本的输入feature为token_ids和label_ids，但是对于BERT，还要求从输入中额外提取position、segment、input_mask等特征，因此经过register后，reader会自动补充backbone所要求的字段
     cls_reader.register_with(ernie)
+    cls_reader2.register_with(ernie)
+    print(cls_reader.outputs_attr)
+<<<<<<< HEAD:test/test2/run.py
+
+    print("preparing data...")
+    print(cls_reader.num_examples)
+    cls_reader.load_data(train_file, batch_size)
+    cls_reader2.load_data(train_file, batch_size)
+=======
+    print(predict_cls_reader.outputs_attr)
 
     print("preparing data...")
     print(cls_reader.num_examples)
     cls_reader.load_data(train_file, batch_size, num_epochs=num_epochs)
+>>>>>>> remotes/upstream/r0.3-api:test/test3/run.py
     print(cls_reader.num_examples)
     print('done!')
-    input_dim = config['hidden_size']
-    num_classes = 2
-    dropout_prob = 0.1
-    random_seed = 1
+
     # 创建任务头（task head），如分类、匹配、机器阅读理解等。每个任务头有跟该任务相关的必选/可选参数。注意，任务头与reader是解耦合的，只要任务头依赖的数据集侧的字段能被reader提供，那么就是合法的
-    cls_head = palm.head.Classify(num_classes, input_dim, dropout_prob)
+    cls_head = palm.head.Classify(4, 1024, 0.1)
+<<<<<<< HEAD:test/test2/run.py
+    cls_head2 = palm.head.Classify(4, 1024, 0.1)
+
+    # 根据reader和任务头来创建一个训练器trainer，trainer代表了一个训练任务，内部维护着训练进程、和任务的关键信息，并完成合法性校验，该任务的模型保存、载入等相关规则控制
+    trainer = palm.Trainer('cls')
+    trainer2 = palm.Trainer('senti_cls')
+    mh_trainer = palm.MultiHeadTrainer([trainer, trainer2])
+=======
 
     # 根据reader和任务头来创建一个训练器trainer，trainer代表了一个训练任务，内部维护着训练进程、和任务的关键信息，并完成合法性校验，该任务的模型保存、载入等相关规则控制
     trainer = palm.Trainer('senti_cls')
+>>>>>>> remotes/upstream/r0.3-api:test/test3/run.py
 
+    # match4mrqa.reuse_head_with(mrc4mrqa)
 
-    loss_var = trainer.build_forward(ernie, cls_head)
+    # data_vars = cls_reader.build()
+    # output_vars = ernie.build(data_vars)
+    # cls_head.build({'backbone': output_vars, 'reader': data_vars})
 
+<<<<<<< HEAD:test/test2/run.py
+    loss_var = mh_trainer.build_forward(ernie, [cls_head, cls_head2])
 
     n_steps = cls_reader.num_examples * num_epochs // batch_size
+    warmup_steps = int(0.1 * n_steps)
+    print(warmup_steps)
+    sched = palm.lr_sched.TriangularSchedualer(warmup_steps, n_steps)
+=======
+    loss_var = trainer.build_forward(ernie, cls_head)
+
+    # controller.build_forward()
+    # Error! a head/backbone can be only build once! Try NOT to call build_forward method for any Trainer!
+
+    # n_steps = cls_reader.num_examples * num_epochs // batch_size
     # warmup_steps = int(0.1 * n_steps)
     # print(warmup_steps)
-    """
     # sched = palm.lr_sched.TriangularSchedualer(warmup_steps, n_steps)
     sched = None
+>>>>>>> remotes/upstream/r0.3-api:test/test3/run.py
 
     adam = palm.optimizer.Adam(loss_var, lr, sched)
 
-    trainer.build_backward(optimizer=adam, weight_decay=0.01)
+    mh_trainer.build_backward(optimizer=adam, weight_decay=0.001)
+    
+    # mh_trainer.random_init_params()
+    mh_trainer.load_pretrain('pretrain/ernie/params')
 
-    trainer.random_init_params()
-    trainer.load_pretrain('pretrain/ernie-ch-uncased-base/params')
+    # trainer.train(iterator_fn, print_steps=1, save_steps=5, save_path='outputs', save_type='ckpt,predict')
+<<<<<<< HEAD:test/test2/run.py
+    mh_trainer.fit_readers_with_mixratio([cls_reader, cls_reader2], 'cls', 2)
+    mh_trainer.train(print_steps=1)
+    # trainer.save()
 
+=======
     trainer.fit_reader(cls_reader)
-    trainer.train(print_steps=1, save_steps=n_steps-24, save_path='outputs', save_type='ckpt')
-    """
-    model_path = './outputs/ckpt.step'+str(n_steps-24)
+    trainer.train(print_steps=1)
+    # trainer.save()
+
     print('prepare to predict...')
-    pred_ernie = palm.backbone.ERNIE.from_config(config, phase='predict')
-    predict_cls_reader.register_with(pred_ernie)
-    predict_cls_reader.load_data(predict_file, 8)
-    
-    cls_pred_head = palm.head.Classify(num_classes, input_dim, phase='predict')
+    pred_ernie = palm.backbone.ERNIE.from_config(config, phase='pred')
+    cls_pred_head = palm.head.Classify(4, 1024, phase='pred')
     trainer.build_predict_forward(pred_ernie, cls_pred_head)
-    pred_ckpt = trainer.load_ckpt(model_path, phase='predict')
-    trainer.fit_reader(predict_cls_reader, phase='predict')
-    
+
+    predict_cls_reader.load_data(predict_file, 8)
     print(predict_cls_reader.num_examples)
-    
+    predict_cls_reader.register_with(pred_ernie)
+    trainer.fit_reader(predict_cls_reader, phase='predict')
     print('predicting..')
-    trainer.predict(print_steps=20, output_dir="outputs/test/")
+    trainer.predict(print_steps=20)
+
+
+
+
+
+
+
+
+    # controller = palm.Controller([mrqa, match4mrqa, mlm4mrqa])
+
+    # loss = controller.build_forward(bb, mask_task=[])
+
+    # n_steps = controller.estimate_train_steps(basetask=mrqa, num_epochs=2, batch_size=8, dev_count=4)
+    # adam = palm.optimizer.Adam(loss)
+    # sched = palm.schedualer.LinearWarmup(learning_rate, max_train_steps=n_steps, warmup_steps=0.1*n_steps)
+    # 
+    # controller.build_backward(optimizer=adam, schedualer=sched, weight_decay=0.001, use_ema=True, ema_decay=0.999)
+
+    # controller.random_init_params()
+    # controller.load_pretrain('../../pretrain_model/ernie/params')
+    # controller.train()
+
+
+
+
+
+    # controller = palm.Controller(config='config.yaml', task_dir='tasks', for_train=False)
+    # controller.pred('mrqa', inference_model_dir='output_model/secondrun/mrqa/infer_model')
+
+
+>>>>>>> remotes/upstream/r0.3-api:test/test3/run.py