提交 79b33748 编写于 作者: K kinghuin 提交者: wuzewu

debug ernie tiny reader

上级 eb72d386
...@@ -49,7 +49,7 @@ class BaseReader(object): ...@@ -49,7 +49,7 @@ class BaseReader(object):
in_tokens=False): in_tokens=False):
self.max_seq_len = max_seq_len self.max_seq_len = max_seq_len
if sp_model_path and word_dict_path: if sp_model_path and word_dict_path:
self.tzokenizer = tokenization.WSSPTokenizer( self.tokenizer = tokenization.WSSPTokenizer(
vocab_path, sp_model_path, word_dict_path, ws=True, lower=True) vocab_path, sp_model_path, word_dict_path, ws=True, lower=True)
else: else:
self.tokenizer = tokenization.FullTokenizer( self.tokenizer = tokenization.FullTokenizer(
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册