fs.py 45.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import multiprocessing

import re
import time
import abc
G
gongweibao 已提交
21
from paddle.fluid import core
22 23 24 25
import functools

import shutil

26
__all__ = []
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44


class ExecuteError(Exception):
    pass


class FSFileExistsError(Exception):
    pass


class FSFileNotExistsError(Exception):
    pass


class FSTimeOut(Exception):
    pass


G
gongweibao 已提交
45 46 47 48
class FSShellCmdAborted(ExecuteError):
    pass


49
class FS:
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
    @abc.abstractmethod
    def ls_dir(self, fs_path):
        raise NotImplementedError

    @abc.abstractmethod
    def is_file(self, fs_path):
        raise NotImplementedError

    @abc.abstractmethod
    def is_dir(self, fs_path):
        raise NotImplementedError

    @abc.abstractmethod
    def is_exist(self, fs_path):
        raise NotImplementedError

    @abc.abstractmethod
    def upload(self, local_path, fs_path):
        raise NotImplementedError

    @abc.abstractmethod
    def download(self, fs_path, local_path):
        raise NotImplementedError

    @abc.abstractmethod
    def mkdirs(self, fs_path):
        raise NotImplementedError

    @abc.abstractmethod
    def delete(self, fs_path):
        raise NotImplementedError

    @abc.abstractmethod
    def need_upload_download(self):
        raise NotImplementedError

    @abc.abstractmethod
    def rename(self, fs_src_path, fs_dst_path):
        raise NotImplementedError

    @abc.abstractmethod
G
gongweibao 已提交
91
    def mv(self, fs_src_path, fs_dst_path, overwrite=False, test_exists=False):
92 93 94 95 96 97 98
        raise NotImplementedError

    @abc.abstractmethod
    def upload_dir(self, local_dir, dest_dir):
        raise NotImplementedError

    @abc.abstractmethod
G
gongweibao 已提交
99
    def list_dirs(self, fs_path):
100 101 102
        raise NotImplementedError

    @abc.abstractmethod
G
gongweibao 已提交
103
    def touch(self, fs_path, exist_ok=True):
104 105
        raise NotImplementedError

106 107 108 109
    @abc.abstractmethod
    def cat(self, fs_path=None):
        raise NotImplementedError

110 111

class LocalFS(FS):
112 113 114 115 116 117
    """
    A tool of local file system.

    Examples:
        .. code-block:: python

118
            from paddle.distributed.fleet.utils import LocalFS
119 120 121 122 123

            client = LocalFS()
            subdirs, files = client.ls_dir("./")
    """

124
    def ls_dir(self, fs_path):
125
        """
126 127 128 129 130 131
        List directorys and files under `fs_path` .

        Args:
            fs_path(str): The local file path.

        Returns:
132
            Tuple: Return a 2-tuple, the first is a list of all its subdirectories,
133 134 135 136 137
            and the second is a list of all its subfiles, e.g. ([subdirname1, subdirname1, ...], [filename1, filename2, ...]).

        Examples:
            .. code-block:: python

138
                from paddle.distributed.fleet.utils import LocalFS
139 140 141 142

                client = LocalFS()
                subdirs, files = client.ls_dir("./")
        """
143 144 145 146 147 148 149 150 151 152 153 154 155 156
        if not self.is_exist(fs_path):
            return [], []

        dirs = []
        files = []
        for f in os.listdir(fs_path):
            if os.path.isdir(fs_path + "/" + f):
                dirs.append(f)
            else:
                files.append(f)

        return dirs, files

    def mkdirs(self, fs_path):
157
        """
158
        Create a local directory.
159 160 161 162 163 164 165

        Args:
            fs_path(str): The local directory path.

        Examples:
            .. code-block:: python

166
                from paddle.distributed.fleet.utils import LocalFS
167 168 169 170 171

                client = LocalFS()
                client.mkdirs("test_mkdirs")
                client.delete("test_mkdirs")
        """
172
        assert not os.path.isfile(fs_path), "{} is already a file".format(
173 174
            fs_path
        )
175 176
        os.system("mkdir -p {}".format(fs_path))

G
gongweibao 已提交
177
    def rename(self, fs_src_path, fs_dst_path):
178 179 180 181 182 183 184 185 186 187
        """
        Rename the file.

        Args:
            fs_src_path(str): The actual name of the file or directory
            fs_dst_path(str): The new name of the file or directory.

        Examples:
            .. code-block:: python

188
                from paddle.distributed.fleet.utils import LocalFS
189 190 191 192 193 194 195 196 197

                client = LocalFS()
                client.touch("test_rename_src")
                print(client.is_exists("test_rename_src")) # True
                client.rename("test_rename_src", "test_rename_dst")
                print(client.is_exists("test_rename_src")) # False
                print(client.is_exists("test_rename_dst")) # True
                client.delete("test_rename_dst")
        """
G
gongweibao 已提交
198
        os.rename(fs_src_path, fs_dst_path)
199 200 201 202 203 204 205 206

    def _rmr(self, fs_path):
        shutil.rmtree(fs_path)

    def _rm(self, fs_path):
        os.remove(fs_path)

    def delete(self, fs_path):
207 208 209 210 211 212 213 214 215
        """
        Delete the local file path, whether it's a file or directory.

        Args:
            fs_path(str): The local file path.

        Examples:
            .. code-block:: python

216
                from paddle.distributed.fleet.utils import LocalFS
217 218 219 220 221

                client = LocalFS()
                client.mkdirs("test_localFS_mkdirs")
                client.delete("test_localFS_mkdirs")
        """
222 223 224 225 226 227 228 229 230 231 232
        if not self.is_exist(fs_path):
            return

        if os.path.isfile(fs_path):
            return self._rm(fs_path)

        return self._rmr(fs_path)

    def need_upload_download(self):
        return False

G
gongweibao 已提交
233
    def is_file(self, fs_path):
234 235 236 237 238 239 240 241 242 243 244 245
        """
        Whether the local file path is a file.

        Args:
            fs_path(str): The local file path.

        Returns:
            Bool: Return true if the path exists and it's a file, otherwise return false.

        Examples:
            .. code-block:: python

246
                from paddle.distributed.fleet.utils import LocalFS
247 248 249 250 251 252

                client = LocalFS()
                client.touch("test_is_file")
                print(client.is_file("test_is_file")) # True
                client.delete("test_is_file")
        """
G
gongweibao 已提交
253 254 255
        return os.path.isfile(fs_path)

    def is_dir(self, fs_path):
256 257 258 259 260 261 262 263 264 265 266 267
        """
        Whether the local file path is a directory.

        Args:
            fs_path(str): The local file path.

        Returns:
            Bool: Return true if the path exists and it's a directory, otherwise return false.

        Examples:
            .. code-block:: python

268
                from paddle.distributed.fleet.utils import LocalFS
269 270 271 272 273 274

                client = LocalFS()
                client.mkdirs("test_is_dir")
                print(client.is_dir("test_is_file")) # True
                client.delete("test_is_dir")
        """
G
gongweibao 已提交
275 276 277
        return os.path.isdir(fs_path)

    def is_exist(self, fs_path):
278 279 280 281 282 283 284
        """
        Whether the local file path exists.

        Args:
            fs_path(str): The local file path.

        Returns:
285
            Bool: Wheter it's a file or directory, return true if the path exists,
286 287 288 289 290
            otherwise return false.

        Examples:
            .. code-block:: python

291
                from paddle.distributed.fleet.utils import LocalFS
292 293 294 295

                client = LocalFS()
                ret = local_fs.is_exist("test_is_exist")
        """
G
gongweibao 已提交
296 297 298
        return os.path.exists(fs_path)

    def touch(self, fs_path, exist_ok=True):
299 300 301 302 303 304 305 306 307 308 309
        """
        Create a local file.

        Args:
            fs_path(str): The local file path.
            exist_ok(bool): When `fs_path` exists, if `exist_ok` is set false,
            program will throw an Exception. Default is true.

        Examples:
            .. code-block:: python

310
                from paddle.distributed.fleet.utils import LocalFS
311 312 313 314 315

                client = LocalFS()
                client.touch("test_touch")
                client.delete("test_touch")
        """
G
gongweibao 已提交
316 317 318 319 320
        if self.is_exist(fs_path):
            if exist_ok:
                return
            raise FSFileExistsError

1
123malin 已提交
321
        os.system("touch {}".format(fs_path))
322

G
gongweibao 已提交
323
    def mv(self, src_path, dst_path, overwrite=False, test_exists=False):
324 325 326 327 328 329 330 331 332 333 334
        """
        Move a local file or directory from `src_path` to `dst_path` .

        Args:
            src_path(str):  Name of the file or directory, that's needed to be moved.
            dst_path(str):  Name of the file or directory to which to move to.
            overwrite(bool): Whether to re-write `dst_path` if that exists. Default is False.

        Examples:
            .. code-block:: python

335
                from paddle.distributed.fleet.utils import LocalFS
336 337 338 339 340 341

                client = LocalFS()
                client.touch("test_mv_src")
                client.mv("test_mv_src", "test_mv_dst")
                client.delete("test_mv_dst")
        """
342 343 344
        if not self.is_exist(src_path):
            raise FSFileNotExistsError

G
gongweibao 已提交
345 346 347
        if overwrite and self.is_exist(dst_path):
            self.delete(dst_path)

348 349 350 351 352
        if self.is_exist(dst_path):
            raise FSFileExistsError

        return self.rename(src_path, dst_path)

G
gongweibao 已提交
353
    def list_dirs(self, fs_path):
354
        """
355 356 357 358 359 360 361 362 363 364 365
        Only list directorys under `fs_path` .

        Args:
            fs_path(str): The local file path.

        Returns:
            List: A list of all its subdirectories, e.g. [subdirname1, subdirname1, ...].

        Examples:
            .. code-block:: python

366
                from paddle.distributed.fleet.utils import LocalFS
367 368 369

                client = LocalFS()
                subdirs = client.list_dirs("./")
G
gongweibao 已提交
370 371 372 373 374 375 376 377 378 379
        """
        if not self.is_exist(fs_path):
            return []

        dirs = [
            f for f in os.listdir(fs_path) if os.path.isdir(fs_path + "/" + f)
        ]

        return dirs

380

G
gongweibao 已提交
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
def _handle_errors(max_time_out=None):
    def decorator(f):
        @functools.wraps(f)
        def handler(*args, **kwargs):
            o = args[0]
            time_out = max_time_out
            if time_out is None:
                time_out = float(o._time_out) / 1000.0
            else:
                time_out /= 1000.0
            inter = float(o._sleep_inter) / 1000.0

            start = time.time()
            last_print_time = start
            while True:
                try:
                    return f(*args, **kwargs)
398
                # important: only ExecuteError need to retry
G
gongweibao 已提交
399 400
                except ExecuteError as e:
                    if time.time() - start >= time_out:
401 402 403 404 405
                        raise FSTimeOut(
                            "args:{} timeout:{}".format(
                                args, time.time() - start
                            )
                        )
G
gongweibao 已提交
406 407 408 409

                    time.sleep(inter)

                if time.time() - last_print_time > 30:
410 411 412 413 414
                    print(
                        "hadoop operator timeout:args:{} timeout:{}".format(
                            args, time.time() - start
                        )
                    )
G
gongweibao 已提交
415 416 417 418 419 420 421
                    last_print_time = time.time()

        return handler

    return decorator


422
class HDFSClient(FS):
423 424 425 426
    """
    A tool of HDFS.

    Args:
427
        hadoop_home(str): Hadoop home.
428 429 430 431 432 433 434
        configs(dict): Hadoop config. It is a dictionary and needs to contain the
            keys: "fs.default.name" and "hadoop.job.ugi".

    Examples:

        .. code-block:: text

435
            from paddle.distributed.fleet.utils import HDFSClient
436 437 438 439 440 441 442 443 444 445 446
            hadoop_home = "/home/client/hadoop-client/hadoop/"

            configs = {
                "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                "hadoop.job.ugi": "hello,hello123"
            }

            client = HDFSClient(hadoop_home, configs)
            client.ls_dir("hdfs:/test_hdfs_client")
    """

447
    def __init__(
448 449 450 451 452 453
        self,
        hadoop_home,
        configs,
        time_out=5 * 60 * 1000,  # ms
        sleep_inter=1000,
    ):  # ms
454 455 456 457 458 459 460
        self.pre_commands = []
        hadoop_bin = '%s/bin/hadoop' % hadoop_home
        self.pre_commands.append(hadoop_bin)
        dfs = 'fs'
        self.pre_commands.append(dfs)

        if configs:
461
            for k, v in configs.items():
G
gongweibao 已提交
462 463
                config_command = '-D%s=%s' % (k, v)
                self.pre_commands.append(config_command)
464 465 466 467 468

        self._time_out = time_out
        self._sleep_inter = sleep_inter
        self._base_cmd = " ".join(self.pre_commands)
        self._bd_err_re = re.compile(
469 470
            r'\s?responseErrorMsg\s?\:.*, errorCode\:\s?[0-9]+, path\:'
        )
471

D
danleifeng 已提交
472
    def _run_cmd(self, cmd, redirect_stderr=False, retry_times=5):
G
gongweibao 已提交
473
        exe_cmd = "{} -{}".format(self._base_cmd, cmd)
D
danleifeng 已提交
474 475 476 477 478 479 480 481 482
        ret = 0
        output = None
        retry_sleep_second = 3
        for x in range(retry_times + 1):
            ret, output = core.shell_execute_cmd(exe_cmd, 0, 0, redirect_stderr)
            ret = int(ret)
            if ret == 0:
                break
            time.sleep(retry_sleep_second)
G
gongweibao 已提交
483 484
        if ret == 134:
            raise FSShellCmdAborted(cmd)
485

G
gongweibao 已提交
486 487 488 489
        return ret, output.splitlines()

    @_handle_errors()
    def list_dirs(self, fs_path):
490
        """
491 492 493 494 495 496 497 498 499 500 501 502
        Only list directorys under `fs_path` .

        Args:
            fs_path(str): The HDFS file path.

        Returns:
            List: A list of all its subdirectories, e.g. [subdirname1, subdirname1, ...].

        Examples:

            .. code-block:: text

503
                from paddle.distributed.fleet.utils import HDFSClient
504 505 506 507 508 509 510 511 512 513

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                subdirs = client.list_dirs("hdfs:/test_hdfs_client")
        """
G
gongweibao 已提交
514 515
        if not self.is_exist(fs_path):
            return []
516

G
gongweibao 已提交
517 518 519 520
        dirs, files = self._ls_dir(fs_path)
        return dirs

    @_handle_errors()
521
    def ls_dir(self, fs_path):
522
        """
523 524 525 526 527 528
        List directorys and files under `fs_path` .

        Args:
            fs_path(str): The HDFS file path.

        Returns:
529
            Tuple: Return a 2-tuple, the first element is the list of all its subdirectories,
530 531 532 533 534 535
            and the second one is the list of all its subfiles, e.g. ([subdirname1, subdirname1, ...], [filename1, filename2, ...]).

        Examples:

            .. code-block:: text

536
                from paddle.distributed.fleet.utils import HDFSClient
537 538 539 540 541 542 543 544 545

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                subdirs, files = client.ls_dir("hdfs:/test_hdfs_client")
546 547 548 549
        """
        if not self.is_exist(fs_path):
            return [], []

G
gongweibao 已提交
550 551 552 553
        return self._ls_dir(fs_path)

    def _ls_dir(self, fs_path):
        cmd = "ls {}".format(fs_path)
554 555 556
        ret, lines = self._run_cmd(cmd)

        if ret != 0:
G
gongweibao 已提交
557
            raise ExecuteError(cmd)
558 559 560 561 562 563 564 565

        dirs = []
        files = []
        for line in lines:
            arr = line.split()
            if len(arr) != 8:
                continue

1
123malin 已提交
566
            p = os.path.basename(arr[7])
567
            if arr[0][0] == 'd':
1
123malin 已提交
568
                dirs.append(p)
569
            else:
1
123malin 已提交
570
                files.append(p)
571 572 573 574 575 576

        return dirs, files

    def _test_match(self, lines):
        for l in lines:
            m = self._bd_err_re.match(l)
577
            if m is not None:
578 579 580 581
                return m

        return None

G
gongweibao 已提交
582
    @_handle_errors()
583
    def is_dir(self, fs_path):
584 585 586 587 588 589 590 591 592 593 594 595 596
        """
        Whether the remote HDFS path is a directory.

        Args:
            fs_path(str): The HDFS file path.

        Returns:
            Bool: Return true if the path exists and it's a directory, otherwise return false.

        Examples:

            .. code-block:: text

597
                from paddle.distributed.fleet.utils import HDFSClient
598 599 600 601 602 603 604 605 606 607

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                ret = client.is_file("hdfs:/test_hdfs_client")
        """
608 609 610
        if not self.is_exist(fs_path):
            return False

G
gongweibao 已提交
611 612 613
        return self._is_dir(fs_path)

    def _is_dir(self, fs_path):
614 615
        cmd = "test -d {}".format(fs_path)
        ret, lines = self._run_cmd(cmd, redirect_stderr=True, retry_times=1)
616 617
        if ret:
            # other error
G
gongweibao 已提交
618
            if self._test_match(lines):
619 620
                print('raise exception: ')
                print('\n'.join(lines))
G
gongweibao 已提交
621
                raise ExecuteError(cmd)
622 623 624 625 626 627

            return False

        return True

    def is_file(self, fs_path):
628 629 630 631 632 633 634 635 636 637 638 639 640
        """
        Whether the remote HDFS path is a file.

        Args:
            fs_path(str): The HDFS file path.

        Returns:
            Bool: Return true if the path exists and it's a file, otherwise return false.

        Examples:

            .. code-block:: text

641
                from paddle.distributed.fleet.utils import HDFSClient
642 643 644 645 646 647 648 649 650 651

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                ret = client.is_file("hdfs:/test_hdfs_client")
        """
652 653 654
        if not self.is_exist(fs_path):
            return False

G
gongweibao 已提交
655
        return not self._is_dir(fs_path)
656

G
gongweibao 已提交
657
    @_handle_errors()
658
    def is_exist(self, fs_path):
659 660 661 662 663 664 665 666 667 668 669 670 671 672
        """
        Whether the remote HDFS path exists.

        Args:
            fs_path(str): The hdfs file path.

        Returns:
            Bool: Whether it's is file or directory, return true if the path exists,
            otherwise return false.

        Examples:

            .. code-block:: text

673
                from paddle.distributed.fleet.utils import HDFSClient
674 675 676 677 678 679 680 681 682 683

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                ret = client.is_exist("hdfs:/test_hdfs_client")
        """
684 685
        cmd = "test -e {} ".format(fs_path)
        ret, out = self._run_cmd(cmd, redirect_stderr=True, retry_times=1)
686
        if ret != 0:
687
            return False
688 689 690

        return True

691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709
    def upload_dir(self, local_dir, dest_dir, overwrite=False):
        """
        upload dir to hdfs
        Args:
            local_dir(str): local dir
            dest_dir(str): hdfs dest dir
            overwrite(bool): is overwrite
        Returns:
            return code
        """
        local_dir = local_dir.rstrip("/")
        dest_dir = dest_dir.rstrip("/")
        local_basename = os.path.basename(local_dir)
        if self.is_exist(dest_dir + "/" + local_basename) and overwrite:
            self.delete(dest_dir + "/" + local_basename)
        if not self.is_exist(dest_dir):
            self.mkdirs(dest_dir)
        self._try_upload(local_dir, dest_dir)

G
gongweibao 已提交
710
    # can't retry
711
    def upload(self, local_path, fs_path, multi_processes=5, overwrite=False):
712 713 714 715 716 717
        """
        Upload the local path to remote HDFS.

        Args:
            local_path(str): The local path.
            fs_path(str): The HDFS path.
718 719
            multi_processes(int|1): the upload data process at the same time, default=5
            overwrite(bool|False): will overwrite file on HDFS or not
720 721 722 723 724

        Examples:

            .. code-block:: text

725
                from paddle.distributed.fleet.utils import HDFSClient
726 727 728 729 730 731 732 733 734 735

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                client.upload("test_hdfs_client", "hdfs:/test_hdfs_client")
        """
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760

        def __subprocess_upload(hdfs_path_single, datas):
            for data in datas:
                self._try_upload(data, hdfs_path_single)

        def get_local_files(path):
            """
            get local files
            Args:
                path(str): local path
            Returns:
                list of local files
            """
            rlist = []

            if not os.path.exists(path):
                return rlist

            if os.path.isdir(path):
                for file in os.listdir(path):
                    t = os.path.join(path, file)
                    rlist.append(t)
            else:
                rlist.append(path)
            return rlist
761 762 763

        local = LocalFS()
        if not local.is_exist(local_path):
G
gongweibao 已提交
764
            raise FSFileNotExistsError("{} not exists".format(local_path))
765

766 767 768 769 770 771 772 773 774 775 776 777
        all_files = get_local_files(local_path)
        if not all_files:
            print("there are nothing need to upload, function exit")
            return

        if self.is_exist(fs_path) and overwrite:
            self.delete(fs_path)
            self.mkdirs(fs_path)

        procs = []
        for i in range(multi_processes):
            process_datas = self._split_files(all_files, i, multi_processes)
778 779 780
            p = multiprocessing.Process(
                target=__subprocess_upload, args=(fs_path, process_datas)
            )
781 782
            procs.append(p)
            p.start()
G
gongweibao 已提交
783

784 785 786
        # complete the processes
        for proc in procs:
            proc.join()
G
gongweibao 已提交
787 788 789 790 791 792

    @_handle_errors()
    def _try_upload(self, local_path, fs_path):
        cmd = "put {} {}".format(local_path, fs_path)
        ret = 0
        try:
793
            ret, _ = self._run_cmd(cmd)
G
gongweibao 已提交
794 795 796 797 798 799 800
            if ret != 0:
                raise ExecuteError(cmd)
        except Exception as e:
            self.delete(fs_path)
            raise e

    # can't retry
801
    def download(self, fs_path, local_path, multi_processes=5, overwrite=False):
802 803 804 805 806 807
        """
        Download remote HDFS path to the local.

        Args:
            fs_path(str):  The HDFS path.
            local_path(str): The local path.
808 809
            multi_processes(int|1): the download data process at the same time, default=1
            overwrite(bool): is overwrite
810 811 812 813 814

        Examples:

            .. code-block:: text

815
                from paddle.distributed.fleet.utils import HDFSClient
816 817 818 819 820 821 822 823 824 825

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                client.download("hdfs:/test_hdfs_client", "./")
        """
826 827 828 829 830 831 832 833 834 835 836

        def __subprocess_download(local_path, datas):
            """
            download file from HDFS
            Args:
                local_path(str): the local file path
                datas(str): the hdfs file path list
            """
            for data in datas:
                self._try_download(data, local_path)

837
        if not self.is_exist(fs_path):
G
gongweibao 已提交
838
            raise FSFileNotExistsError("{} not exits".format(fs_path))
839 840 841 842
        # download file
        if self.is_file(fs_path):
            return self._try_download(fs_path, local_path)
        # download dir
G
gongweibao 已提交
843 844 845
        dirs, all_filenames = self.ls_dir(fs_path)
        all_files = [fs_path + "/" + i for i in all_filenames]
        all_files.extend([fs_path + "/" + i for i in dirs])
846 847 848
        procs = []
        for i in range(multi_processes):
            process_datas = self._split_files(all_files, i, multi_processes)
849 850 851
            p = multiprocessing.Process(
                target=__subprocess_download, args=(local_path, process_datas)
            )
852 853 854 855 856 857
            procs.append(p)
            p.start()

        # complete the processes
        for proc in procs:
            proc.join()
G
gongweibao 已提交
858 859 860 861 862 863

    @_handle_errors()
    def _try_download(self, fs_path, local_path):
        cmd = "get {} {}".format(fs_path, local_path)
        ret = 0
        try:
864
            ret, _ = self._run_cmd(cmd)
G
gongweibao 已提交
865 866 867 868 869 870 871 872
            if ret != 0:
                raise ExecuteError(cmd)
        except Exception as e:
            local_fs = LocalFS()
            local_fs.delete(local_path)
            raise e

    @_handle_errors()
873
    def mkdirs(self, fs_path):
874 875 876 877 878 879 880 881 882 883
        """
        Create a remote HDFS directory.

        Args:
            fs_path(str): The HDFS directory path.

        Examples:

            .. code-block:: text

884
                from paddle.distributed.fleet.utils import HDFSClient
885 886 887 888 889 890 891 892 893 894

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                client.mkdirs("hdfs:/test_hdfs_client")
        """
895 896 897
        if self.is_exist(fs_path):
            return

G
gongweibao 已提交
898 899 900 901
        out_hdfs = False

        cmd = "mkdir {} ".format(fs_path)
        ret, out = self._run_cmd(cmd, redirect_stderr=True)
902
        if ret != 0:
G
gongweibao 已提交
903 904 905 906 907 908 909 910 911
            for l in out:
                if "No such file or directory" in l:
                    out_hdfs = True
                    break
            if not out_hdfs:
                raise ExecuteError(cmd)

        if out_hdfs and not self.is_exist(fs_path):
            cmd = "mkdir -p {}".format(fs_path)
912
            ret, _ = self._run_cmd(cmd)
G
gongweibao 已提交
913 914 915 916
            if ret != 0:
                raise ExecuteError(cmd)

    def mv(self, fs_src_path, fs_dst_path, overwrite=False, test_exists=True):
917 918 919 920 921 922 923
        """
        Move a remote HDFS file or directory from `fs_src_path` to `fs_dst_path` .

        Args:
            fs_src_path(str):  Name of the file or directory, that's needed to be moved.
            fs_dst_path(str):  Name of the file or directory to which to move to.
            overwrite(bool): Whether to re-write `fs_dst_path` if that exists. Default is False.
924
            test_exists(bool): Check the existence of `fs_src_path` and `fs_dst_path` . When `test_exists` is set true, if `fs_src_path` doesn't exist or `fs_dst_path` exists, program will throw an Excetption.
925 926 927 928 929

        Examples:

            .. code-block:: text

930
                from paddle.distributed.fleet.utils import HDFSClient
931 932 933 934 935 936 937 938 939 940

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                client.mv("hdfs:/test_hdfs_client", "hdfs:/test_hdfs_client2")
        """
G
gongweibao 已提交
941 942
        if overwrite and self.is_exist(fs_dst_path):
            self.delete(fs_dst_path)
943 944 945

        if test_exists:
            if not self.is_exist(fs_src_path):
946
                raise FSFileNotExistsError(
947 948
                    "{} is not exists".format(fs_src_path)
                )
949 950

            if self.is_exist(fs_dst_path):
Z
zhangchunle 已提交
951
                raise FSFileExistsError("{} exists already".format(fs_dst_path))
G
gongweibao 已提交
952 953 954 955 956 957 958 959

        return self._try_mv(fs_src_path, fs_dst_path)

    @_handle_errors()
    def _try_mv(self, fs_src_path, fs_dst_path):
        cmd = "mv {} {}".format(fs_src_path, fs_dst_path)
        ret = 0
        try:
960
            ret, _ = self._run_cmd(cmd, retry_times=1)
G
gongweibao 已提交
961 962 963
            if ret != 0:
                raise ExecuteError(cmd)
        except Exception as e:
964
            if not self.is_exist(fs_src_path) and self.is_exist(fs_dst_path):
G
gongweibao 已提交
965 966
                return
            raise e
967 968

    def _rmr(self, fs_path):
G
gongweibao 已提交
969
        cmd = "rmr {}".format(fs_path)
970 971
        ret, _ = self._run_cmd(cmd)
        if ret != 0:
G
gongweibao 已提交
972
            raise ExecuteError(cmd)
973 974

    def _rm(self, fs_path):
G
gongweibao 已提交
975
        cmd = "rm {}".format(fs_path)
976 977
        ret, _ = self._run_cmd(cmd)
        if ret != 0:
G
gongweibao 已提交
978
            raise ExecuteError(cmd)
979

G
gongweibao 已提交
980
    @_handle_errors()
981
    def delete(self, fs_path):
982 983 984 985 986 987 988 989 990 991
        """
        Delete a remote HDFS path, whether it's a file or directory.

        Args:
            fs_path(str): The HDFS file path.

        Examples:

            .. code-block:: text

992
                from paddle.distributed.fleet.utils import HDFSClient
993 994 995 996 997 998 999 1000 1001 1002

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                client.delete("hdfs:/test_hdfs_client")
        """
1003 1004 1005
        if not self.is_exist(fs_path):
            return

G
gongweibao 已提交
1006
        is_dir = self._is_dir(fs_path)
1007 1008 1009 1010 1011
        if is_dir:
            return self._rmr(fs_path)

        return self._rm(fs_path)

G
gongweibao 已提交
1012
    def touch(self, fs_path, exist_ok=True):
1013 1014 1015 1016 1017
        """
        Create a remote HDFS file.

        Args:
            fs_path(str): The HDFS file path.
1018 1019
            exist_ok(bool): When `fs_path` exists, if `exist_ok` is set false,
            program will throw an Exception. Default is true.
1020 1021 1022 1023 1024

        Examples:

            .. code-block:: text

1025
                from paddle.distributed.fleet.utils import HDFSClient
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                client.touch("hdfs:/test_hdfs_client")
        """
G
gongweibao 已提交
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047
        if self.is_exist(fs_path):
            if exist_ok:
                return
            raise FSFileExistsError

        return self._touchz(fs_path)

    @_handle_errors()
    def _touchz(self, fs_path):
        cmd = "touchz {}".format(fs_path)
        ret, _ = self._run_cmd(cmd)
        if ret != 0:
1048
            raise ExecuteError(cmd)
G
gongweibao 已提交
1049

1050 1051
    def need_upload_download(self):
        return True
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086

    def cat(self, fs_path=None):
        """
        Cat a remote HDFS file.

        Args:
            fs_path(str): The HDFS file path.

        Returns:
            file content

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import HDFSClient

                hadoop_home = "/home/client/hadoop-client/hadoop/"
                configs = {
                    "fs.default.name": "hdfs://xxx.hadoop.com:54310",
                    "hadoop.job.ugi": "hello,hello123"
                }

                client = HDFSClient(hadoop_home, configs)
                client.cat("hdfs:/test_hdfs_client")
        """
        if self.is_file(fs_path):
            output = self._try_cat(fs_path)
            return "\n".join(output)
        else:
            return ""

    @_handle_errors()
    def _try_cat(self, fs_path):
        cmd = "cat {}".format(fs_path)
1087
        ret, output = self._run_cmd(cmd, retry_times=1)
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
        if ret != 0:
            raise ExecuteError(cmd)
        return output

    def _split_files(self, files, trainer_id, trainers):
        """
        split file list
        Args:
            files(list): file list
            trainer_id(int): trainer mpi rank id
            trainers(int): all trainers num
        Returns:
            fileist(list): file list of current trainer
        """
        remainder = len(files) % trainers
        blocksize = len(files) // trainers

        blocks = [blocksize] * trainers
        for i in range(remainder):
            blocks[i] += 1

        trainer_files = [[]] * trainers
        begin = 0
        for i in range(trainers):
1112
            trainer_files[i] = files[begin : begin + blocks[i]]
1113 1114 1115
            begin += blocks[i]

        return trainer_files[trainer_id]
Y
yaoxuefeng 已提交
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129

    def list_files_info(self, path_list):
        """
        list_files return file path and size
        Args:
            path_list(list): file list
        Returns:
            fileist(list): file list with file path and size
        """
        if len(path_list) <= 0:
            return []

        file_list = []

1130
        # concat filelist can speed up 'hadoop ls'
Y
yaoxuefeng 已提交
1131 1132 1133
        str_concat = ""
        for path in path_list:
            str_concat += path + " "
1134 1135 1136
        cmd = (
            "ls " + str_concat + " | awk '{if ($8 != \"\") {print $5\" \"$8 }}'"
        )
Y
yaoxuefeng 已提交
1137
        ret, lines = self._run_cmd(cmd)
1138
        if len(lines) == 0:
Y
yaoxuefeng 已提交
1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149
            logger.warning("list_files empty, path[%s]" % path_list)
            return []
        for line in lines:
            arr = line.split(' ')
            if len(arr) < 2:
                continue
            file_path = arr[1]
            file_size = int(arr[0])
            file_list.append({'path': file_path, 'size': file_size})

        return file_list
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165


class AFSClient(FS):
    """
    A tool of AFS. Use AfsWrapper.

    Examples:

        .. code-block:: text

            from paddle.distributed.fleet.utils import AFSClient
            client = AFSClient()
            client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
            client.ls_dir("hdfs:/test_hdfs_client")
    """

1166
    def __init__(self, time_out=5 * 60 * 1000, sleep_inter=1000):  # ms  # ms
1167 1168 1169 1170 1171 1172 1173
        self._fs = core.AfsWrapper()
        self._time_out = time_out

    def init(self, fs_name, fs_user, fs_passwd, fs_conf):
        self._fs.init(fs_name, fs_user, fs_passwd, fs_conf)

    def list_dirs(self, fs_path):
1174
        """
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
        Only list directorys under `fs_path` .

        Args:
            fs_path(str): The HDFS file path.

        Returns:
            List: A list of all its subdirectories, e.g. [subdirname1, subdirname1, ...].

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                subdirs = client.list_dirs("hdfs:/test_hdfs_client")
        """
        if not self.is_exist(fs_path):
            return []
        # TODO:fengdanlei
        dirs, files = self._ls_dir(fs_path)
        return dirs

    def ls_dir(self, fs_path):
1200
        """
1201 1202 1203 1204 1205 1206
        List directorys and files under `fs_path` .

        Args:
            fs_path(str): The HDFS file path.

        Returns:
1207
            Tuple: Return a 2-tuple, the first element is the list of all its subdirectories,
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
            and the second one is the list of all its subfiles, e.g. ([subdirname1, subdirname1, ...], [filename1, filename2, ...]).

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                subdirs, files = client.ls_dir("hdfs:/test_hdfs_client")
        """
        if not self.is_exist(fs_path):
            return [], []

        return self._ls_dir(fs_path)

    def _ls_dir(self, fs_path):

        files = self._fs.list(fs_path)
        dirs = [fs_path]
        return dirs, files

    def is_dir(self, fs_path):
        """
        Whether the remote HDFS path is a directory.

        Args:
            fs_path(str): The HDFS file path.

        Returns:
            Bool: Return true if the path exists and it's a directory, otherwise return false.

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                ret = client.is_file("hdfs:/test_hdfs_client")
        """
        if not self.is_exist(fs_path):
            return False

        return self._is_dir(fs_path)

    def _is_dir(self, fs_path):
        list_path = self._fs.list(fs_path)
        if (len(list_path)) > 0:
            return True
        else:
            return False

    def is_file(self, fs_path):
        """
        Whether the remote HDFS path is a file.

        Args:
            fs_path(str): The HDFS file path.

        Returns:
            Bool: Return true if the path exists and it's a file, otherwise return false.

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                ret = client.is_file("hdfs:/test_hdfs_client")
        """
        if not self.is_exist(fs_path):
            return False

        return not self._is_dir(fs_path)

    def is_exist(self, fs_path):
        """
        Whether the remote HDFS path exists.

        Args:
            fs_path(str): The hdfs file path.

        Returns:
            Bool: Whether it's is file or directory, return true if the path exists,
            otherwise return false.

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                ret = client.is_exist("hdfs:/test_hdfs_client")
        """
        return self._fs.exist(fs_path)

    def upload_dir(self, local_dir, dest_dir, overwrite=False):
        """
        upload dir to hdfs
        Args:
            local_dir(str): local dir
            dest_dir(str): hdfs dest dir
            overwrite(bool): is overwrite
        Returns:
            return code
        """
        local_dir = local_dir.rstrip("/")
        dest_dir = dest_dir.rstrip("/")
        local_basename = os.path.basename(local_dir)
        if self.is_exist(dest_dir + "/" + local_basename) and overwrite:
            self.delete(dest_dir + "/" + local_basename)
        if not self.is_exist(dest_dir):
            self.mkdirs(dest_dir)
        self._fs.upload(local_dir, dest_dir)

    # can't retry
    def upload(self, local_path, fs_path, multi_processes=1, overwrite=False):
        """
        Upload the local path to remote HDFS.

        Args:
            local_path(str): The local path.
            fs_path(str): The HDFS path.
            multi_processes(int|1): the upload data process at the same time, default=5
            overwrite(bool|False): will overwrite file on HDFS or not

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                client.upload("test_hdfs_client", "hdfs:/test_hdfs_client")
        """

        local = LocalFS()
        if not local.is_exist(local_path):
            raise FSFileNotExistsError("{} not exists".format(local_path))

        self._fs.upload(local_path, fs_path)

    def download(self, fs_path, local_path, multi_processes=1, overwrite=False):
        """
        Download remote HDFS path to the local.

        Args:
            fs_path(str):  The HDFS path.
            local_path(str): The local path.
            multi_processes(int|1): the download data process at the same time, default=1
            overwrite(bool): is overwrite

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                client.download("hdfs:/test_hdfs_client", "./")
        """

        def __subprocess_download(local_path, datas):
            """
            download file from HDFS
            Args:
                local_path(str): the local file path
                datas(str): the hdfs file path list
            """
            for data in datas:
                self._fs.download(local_path, data)

        if not self.is_exist(fs_path):
            raise FSFileNotExistsError("{} not exits".format(fs_path))
        # download file
        if self.is_file(fs_path):
            return self._fs.download(local_path, fs_path)
        # download dir
        _, all_filenames = self.ls_dir(fs_path)
        all_files = [fs_path + i for i in all_filenames]
        procs = []
        for i in range(multi_processes):
            process_datas = self._split_files(all_files, i, multi_processes)
1400 1401 1402
            p = multiprocessing.Process(
                target=__subprocess_download, args=(local_path, process_datas)
            )
1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
            procs.append(p)
            p.start()

        # complete the processes
        for proc in procs:
            proc.join()

    def mkdirs(self, fs_path):
        """
        Create a remote HDFS directory.

        Args:
            fs_path(str): The HDFS directory path.

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                client.mkdirs("hdfs:/test_hdfs_client")
        """
        if self.is_exist(fs_path):
            return
        self._fs.mkdir(fs_path)

    def mv(self, fs_src_path, fs_dst_path, overwrite=False, test_exists=True):
        """
        Move a remote HDFS file or directory from `fs_src_path` to `fs_dst_path` .

        Args:
            fs_src_path(str):  Name of the file or directory, that's needed to be moved.
            fs_dst_path(str):  Name of the file or directory to which to move to.
            overwrite(bool): Whether to re-write `fs_dst_path` if that exists. Default is False.
1439
            test_exists(bool): Check the existence of `fs_src_path` and `fs_dst_path` . When `test_exists` is set true, if `fs_src_path` doesn't exist or `fs_dst_path` exists, program will throw an Excetption.
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                client.mv("hdfs:/test_hdfs_client", "hdfs:/test_hdfs_client2")
        """
        if overwrite and self.is_exist(fs_dst_path):
            self.delete(fs_dst_path)

        if test_exists:
            if not self.is_exist(fs_src_path):
1456
                raise FSFileNotExistsError(
1457 1458
                    "{} is not exists".format(fs_src_path)
                )
1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561

            if self.is_exist(fs_dst_path):
                raise FSFileExistsError("{} exists already".format(fs_dst_path))

        self._fs.mv(fs_src_path, fs_dst_path)

    def delete(self, fs_path):
        """
        Delete a remote HDFS path, whether it's a file or directory.

        Args:
            fs_path(str): The HDFS file path.

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import HDFSClient

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                client.delete("hdfs:/test_hdfs_client")
        """
        if not self.is_exist(fs_path):
            return
        self._fs.remove(fs_path)

    def touch(self, fs_path, exist_ok=True):
        """
        Create a remote HDFS file.

        Args:
            fs_path(str): The HDFS file path.
            exist_ok(bool): When `fs_path` exists, if `exist_ok` is set false,
            program will throw an Exception. Default is true.

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                client.touch("hdfs:/test_hdfs_client")
        """
        if self.is_exist(fs_path):
            if exist_ok:
                return
            raise FSFileExistsError

        return self._fs.touchz(fs_path)

    def need_upload_download(self):
        return True

    def cat(self, fs_path=None):
        """
        Cat a remote HDFS file.

        Args:
            fs_path(str): The HDFS file path.

        Returns:
            file content

        Examples:

            .. code-block:: text

                from paddle.distributed.fleet.utils import AFSClient

                client = AFSClient()
                client.init("hdfs://xxx.hadoop.com:54310", "hello", "hello123", "./fs_conf")
                client.cat("hdfs:/test_hdfs_client")
        """
        if self.is_file(fs_path):
            return self._fs.cat(fs_path)
        else:
            return ""

    def _split_files(self, files, trainer_id, trainers):
        """
        split file list
        Args:
            files(list): file list
            trainer_id(int): trainer mpi rank id
            trainers(int): all trainers num
        Returns:
            fileist(list): file list of current trainer
        """
        remainder = len(files) % trainers
        blocksize = len(files) // trainers

        blocks = [blocksize] * trainers
        for i in range(remainder):
            blocks[i] += 1

        trainer_files = [[]] * trainers
        begin = 0
        for i in range(trainers):
1562
            trainer_files[i] = files[begin : begin + blocks[i]]
1563 1564 1565
            begin += blocks[i]

        return trainer_files[trainer_id]