From db8d88905d7c6f75b421ac6585854867fe508ae3 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 25 Jul 2018 02:09:03 +0000 Subject: [PATCH] Fix bug of load_lines --- fluid/neural_machine_translation/transformer/reader.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fluid/neural_machine_translation/transformer/reader.py b/fluid/neural_machine_translation/transformer/reader.py index c19ac742..604d1ddc 100644 --- a/fluid/neural_machine_translation/transformer/reader.py +++ b/fluid/neural_machine_translation/transformer/reader.py @@ -231,8 +231,7 @@ class DataReader(object): lens.append(len(src_trg_ids[1])) self._sample_infos.append(SampleInfo(i, max(lens), min(lens))) - @staticmethod - def _load_lines(fpattern, tar_fname): + def _load_lines(self, fpattern, tar_fname): fpaths = glob.glob(fpattern) if len(fpaths) == 1 and tarfile.is_tarfile(fpaths[0]): @@ -241,7 +240,7 @@ class DataReader(object): f = tarfile.open(fpaths[0], 'r') for line in f.extractfile(tar_fname): - yield line.split() + yield line.split(self._delimiter) else: for fpath in fpaths: if not os.path.isfile(fpath): @@ -249,7 +248,7 @@ class DataReader(object): with open(fpath, 'r') as f: for line in f: - yield line.split() + yield line.split(self._delimiter) @staticmethod def load_dict(dict_path, reverse=False): -- GitLab