From 9757e98fbe146e6b2bcb55284baa0333e0c9e285 Mon Sep 17 00:00:00 2001
From: chenxuyi <chenxuyi@baidu.com>
Date: Wed, 20 May 2020 12:03:05 +0800
Subject: [PATCH] + tokenizer BOS

---
 ernie/tokenizing_ernie.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ernie/tokenizing_ernie.py b/ernie/tokenizing_ernie.py
index 428138f..4556bb8 100644
--- a/ernie/tokenizing_ernie.py
+++ b/ernie/tokenizing_ernie.py
@@ -79,6 +79,8 @@ class ErnieTokenizer(object):
         'ernie-2.0-en': bce + 'model-ernie2.0-en.1.tar.gz',
         'ernie-2.0-large-en':  bce + 'model-ernie2.0-large-en.1.tar.gz',
         'ernie-tiny': bce + 'model-ernie_tiny.1.tar.gz',
+        'ernie-gen-base-en': bce + 'model-ernie-gen-base-en.1.tar.gz',
+        'ernie-gen-large-en': bce + 'model-ernie-gen-large-en.1.tar.gz',
     }
     @classmethod
     def from_pretrained(cls, pretrain_dir_or_url, force_download=False, **kwargs):
-- 
GitLab