未验证 提交 90e2f2d6 编写于 作者: littletomatodonkey's avatar littletomatodonkey 提交者: GitHub

fix nlp return token_type_ids (#7289)

上级 a40f64a7
...@@ -918,7 +918,10 @@ class VQATokenLabelEncode(object): ...@@ -918,7 +918,10 @@ class VQATokenLabelEncode(object):
text = info["text"] text = info["text"]
encode_res = self.tokenizer.encode( encode_res = self.tokenizer.encode(
text, pad_to_max_seq_len=False, return_attention_mask=True) text,
pad_to_max_seq_len=False,
return_attention_mask=True,
return_token_type_ids=True)
if not self.add_special_ids: if not self.add_special_ids:
# TODO: use tok.all_special_ids to remove # TODO: use tok.all_special_ids to remove
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册