未验证 提交 34d70094 编写于 作者: Q Qiyang Min 提交者: GitHub

Merge pull request #1277 from velconia/fix_transformer

Fix the reader in transformer of python3
import glob
import six
import os
import tarfile
......@@ -262,8 +263,10 @@ class DataReader(object):
if not os.path.isfile(fpath):
raise IOError("Invalid file: %s" % fpath)
with open(fpath, "r") as f:
with open(fpath, "rb") as f:
for line in f:
if six.PY3:
line = line.decode()
fields = line.strip("\n").split(self._field_delimiter)
if (not self._only_src and len(fields) == 2) or (
self._only_src and len(fields) == 1):
......@@ -272,8 +275,10 @@ class DataReader(object):
@staticmethod
def load_dict(dict_path, reverse=False):
word_dict = {}
with open(dict_path, "r") as fdict:
with open(dict_path, "rb") as fdict:
for idx, line in enumerate(fdict):
if six.PY3:
line = line.decode()
if reverse:
word_dict[idx] = line.strip("\n")
else:
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册