From 66f4b29220b1417ba65f25d9636eba84d280cc13 Mon Sep 17 00:00:00 2001 From: danleifeng <52735331+danleifeng@users.noreply.github.com> Date: Thu, 21 Oct 2021 15:23:17 +0800 Subject: [PATCH] fix hdfs download_dir (#36590) --- python/paddle/distributed/fleet/utils/fs.py | 4 ++-- python/paddle/fluid/tests/unittests/hdfs_test_utils.py | 2 +- python/paddle/fluid/tests/unittests/test_hdfs3.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/paddle/distributed/fleet/utils/fs.py b/python/paddle/distributed/fleet/utils/fs.py index f56580f8ca2..8895a529526 100644 --- a/python/paddle/distributed/fleet/utils/fs.py +++ b/python/paddle/distributed/fleet/utils/fs.py @@ -842,8 +842,8 @@ class HDFSClient(FS): if self.is_file(fs_path): return self._try_download(fs_path, local_path) # download dir - _, all_files = self.ls_dir(fs_path) - + _, all_filenames = self.ls_dir(fs_path) + all_files = [fs_path + i for i in all_filenames] procs = [] for i in range(multi_processes): process_datas = self._split_files(all_files, i, multi_processes) diff --git a/python/paddle/fluid/tests/unittests/hdfs_test_utils.py b/python/paddle/fluid/tests/unittests/hdfs_test_utils.py index 6b490490739..69ccc7088b8 100644 --- a/python/paddle/fluid/tests/unittests/hdfs_test_utils.py +++ b/python/paddle/fluid/tests/unittests/hdfs_test_utils.py @@ -195,7 +195,7 @@ class FSTestBase(unittest.TestCase): fs.download(src_file, dst_file) local = LocalFS() - self.assertTrue(local.is_exist(dst_file)) + self.assertTrue(local.is_exist(file1)) local.delete(dst_file) fs.delete(src_file) diff --git a/python/paddle/fluid/tests/unittests/test_hdfs3.py b/python/paddle/fluid/tests/unittests/test_hdfs3.py index d214768b2e3..57b0b1ba45f 100644 --- a/python/paddle/fluid/tests/unittests/test_hdfs3.py +++ b/python/paddle/fluid/tests/unittests/test_hdfs3.py @@ -40,6 +40,7 @@ class FSTest3(FSTestBase): self._test_upload(fs) self._test_upload_dir(fs) self._test_download(fs) + self._test_download_dir(fs) def test_local(self): fs = LocalFS() -- GitLab