From 881bd97834e47e890e661219feba8a83a3a15635 Mon Sep 17 00:00:00 2001 From: Weiyue Su Date: Mon, 30 Nov 2020 17:54:01 +0800 Subject: [PATCH] local Path in from_pretrained (#600) Co-authored-by: suweiyue --- ernie/modeling_ernie.py | 2 +- ernie/tokenizing_ernie.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ernie/modeling_ernie.py b/ernie/modeling_ernie.py index 781c93b..415e956 100644 --- a/ernie/modeling_ernie.py +++ b/ernie/modeling_ernie.py @@ -202,7 +202,7 @@ class PretrainedModel(object): pretrain_dir = _fetch_from_remote(url, force_download) else: log.info('pretrain dir %s not in %s, read from local' % (pretrain_dir_or_url, repr(cls.resource_map))) - pretrain_dir = pretrain_dir_or_url + pretrain_dir = Path(pretrain_dir_or_url) if not pretrain_dir.exists(): raise ValueError('pretrain dir not found: %s' % pretrain_dir) diff --git a/ernie/tokenizing_ernie.py b/ernie/tokenizing_ernie.py index 8feef46..2733767 100644 --- a/ernie/tokenizing_ernie.py +++ b/ernie/tokenizing_ernie.py @@ -24,6 +24,10 @@ import re import logging import tempfile from functools import partial +if six.PY2: + from pathlib2 import Path +else: + from pathlib import Path from tqdm import tqdm import numpy as np @@ -90,7 +94,7 @@ class ErnieTokenizer(object): pretrain_dir = _fetch_from_remote(url, force_download=force_download) else: log.info('pretrain dir %s not in %s, read from local' % (pretrain_dir_or_url, repr(cls.resource_map))) - pretrain_dir = pretrain_dir_or_url + pretrain_dir = Path(pretrain_dir_or_url) if not pretrain_dir.exists(): raise ValueError('pretrain dir not found: %s' % pretrain_dir) vocab_path = pretrain_dir / 'vocab.txt' @@ -206,7 +210,7 @@ class ErnieTinyTokenizer(ErnieTokenizer): pretrain_dir = _fetch_from_remote(url, force_download) else: log.info('pretrain dir %s not in %s, read from local' % (pretrain_dir_or_url, repr(cls.resource_map))) - pretrain_dir = pretrain_dir_or_url + pretrain_dir = Path(pretrain_dir_or_url) if not pretrain_dir.exists(): raise ValueError('pretrain dir not found: %s' % pretrain_dir) vocab_path = pretrain_dir / 'vocab.txt' -- GitLab