diff --git a/demo/finetune_classifier.py b/demo/finetune_classifier.py index 3c189050b33481385d567f1274d050a183aa903e..7f610e9755638aa03231d55e280f67bc38e7d764 100644 --- a/demo/finetune_classifier.py +++ b/demo/finetune_classifier.py @@ -25,7 +25,6 @@ from functools import reduce, partial import numpy as np import multiprocessing import tempfile -import jieba import re import paddle diff --git a/demo/finetune_classifier_dygraph.py b/demo/finetune_classifier_dygraph.py index 53764aadc2d32191221e39cfac6f65fb9e415399..d82eadeb32799f605f76fbc202aa3125a101c71a 100644 --- a/demo/finetune_classifier_dygraph.py +++ b/demo/finetune_classifier_dygraph.py @@ -58,6 +58,7 @@ if __name__ == '__main__': parser.add_argument('--save_dir', type=str, default=None, help='model output directory') parser.add_argument('--max_steps', type=int, default=None, help='max_train_steps, set this to EPOCH * NUM_SAMPLES / BATCH_SIZE') parser.add_argument('--wd', type=float, default=0.01, help='weight decay, aka L2 regularizer') + parser.add_argument('--init_checkpoint', type=str, default=None, help='checkpoint to warm start from') args = parser.parse_args() @@ -103,6 +104,11 @@ if __name__ == '__main__': with FD.guard(place): model = ErnieModelForSequenceClassification.from_pretrained(args.from_pretrained, num_labels=3, name='') + if args.init_checkpoint is not None: + log.info('loading checkpoint from %s' % args.init_checkpoint) + sd, _ = FD.load_dygraph(args.init_checkpoint) + model.set_dict(sd) + g_clip = F.clip.GradientClipByGlobalNorm(1.0) #experimental if args.use_lr_decay: opt = AdamW(learning_rate=LinearDecay(args.lr, int(args.warmup_proportion * args.max_steps), args.max_steps), parameter_list=model.parameters(), weight_decay=args.wd, grad_clip=g_clip) diff --git a/demo/pretrain/make_pretrain_data.py b/demo/pretrain/make_pretrain_data.py index 96be1d479e85d9f09da502cc87b0000b4cb7218e..6f3d6ed90d48555144ce074dc92ec79ad13eb73a 100644 --- a/demo/pretrain/make_pretrain_data.py +++ b/demo/pretrain/make_pretrain_data.py @@ -1,7 +1,6 @@ import sys import argparse import struct -#import jieba import random as r import re import gzip diff --git a/demo/pretrain/pretrain.py b/demo/pretrain/pretrain.py index 7d03e2aad94ca89ae81fc60c007691bc3fd54e23..d3ba1c92f584bc7866e05b43d94478edc451af28 100644 --- a/demo/pretrain/pretrain.py +++ b/demo/pretrain/pretrain.py @@ -44,7 +44,6 @@ from ernie.optimization import optimization import propeller.paddle as propeller from propeller.paddle.data import Dataset -#import jieba from propeller import log log.setLevel(logging.DEBUG) diff --git a/demo/pretrain/pretrain_dygraph.py b/demo/pretrain/pretrain_dygraph.py index f32e37aa4bf30ea5f603ab244355a57cf00e2381..3911ad9ae529c13eb89bd03a43e56c35e83b71f2 100644 --- a/demo/pretrain/pretrain_dygraph.py +++ b/demo/pretrain/pretrain_dygraph.py @@ -45,7 +45,6 @@ from ernie.optimization import AdamW, LinearDecay import propeller.paddle as propeller from propeller.paddle.data import Dataset -#import jieba from propeller import log log.setLevel(logging.DEBUG) diff --git a/ernie/tokenizing_ernie.py b/ernie/tokenizing_ernie.py index 4556bb8c239167a2b6804e48b27f0b6de15b0199..25ff1f5353390557768269e249f01fd7b849131e 100644 --- a/ernie/tokenizing_ernie.py +++ b/ernie/tokenizing_ernie.py @@ -222,14 +222,14 @@ class ErnieTinyTokenizer(ErnieTokenizer): def __init__(self, vocab, sp_model_path, **kwargs): super(ErnieTinyTokenizer, self).__init__(vocab, **kwargs) import sentencepiece as spm + import jieba as jb self.sp_model = spm.SentencePieceProcessor() self.window_size = 5 self.sp_model.Load(sp_model_path) - from LAC import LAC - self.lac = LAC() + self.jb = jb def cut(self, sentence): - return self.lac.lexer(sentence) + return self.jb.cut(sentence) def tokenize(self, text): if len(text) == 0: diff --git a/requirements.txt b/requirements.txt index 23fc00f0a84ae5e16c51daea7aa4ec7593998eec..cacf671c0f65f21a48ada5396afce164c10d1437 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ pyzmq==18.0.2 six==1.11.0 sklearn==0.0 sentencepiece==0.1.8 -LAC +jieba==0.39