diff --git a/doc/design/file_mananger/README.md b/doc/design/file_mananger/README.md index 6078696e6e2ff3c7000b5a2d0d306e4593a7c3ea..bca8f48b8e252f0b3d6ba8e51f14b27eded4d804 100644 --- a/doc/design/file_mananger/README.md +++ b/doc/design/file_mananger/README.md @@ -2,7 +2,7 @@ ## Objetive 在本文档中，我们设计说明了用户上传、下载、管理自己在PaddlePaddle Cloud上的文件所涉及到的模块和流程 - + ## Module ### PFS Client @@ -25,7 +25,7 @@ - 在kubernets中运行 - [RESTAPI](./RESTAPI.md)接口 -## 流程 +## 解释 ### 为什么有chunk的抽象： 用户文件可能是比较大的，上传到Cloud或者下载到本地的时间可能比较长，而且在传输的过程中也可能出现网络不稳定的情况。为了应对以上的问题，我们提出了chunk的概念，一个chunk由所在的文件偏移、数据、数据长度及校验值组成。文件数据内容的上传和下载都是都过chunk的操作来实现的。由于chunk比较小（默认256K），完成一个传输动作的transaction的时间也比较短，不容易出错。 diff --git a/doc/design/file_mananger/RESTAPI.md b/doc/design/file_mananger/RESTAPI.md index 4286195970bbbd1f8e4d2d9328f3dba597e8fdea..8d0f37be9958290a63080151a58145de79904126 100644 --- a/doc/design/file_mananger/RESTAPI.md +++ b/doc/design/file_mananger/RESTAPI.md @@ -1,4 +1,4 @@ -# REST API +# REST API Interface - file ``` diff --git a/doc/design/file_mananger/pfs/cp.md b/doc/design/file_mananger/pfs/cp.md index fcc57258a1763849f051b95f9946b85e0ebaf842..af4204d717ffbdb4ee84ad20201fc85a3cbe0f21 100644 --- a/doc/design/file_mananger/pfs/cp.md +++ b/doc/design/file_mananger/pfs/cp.md @@ -1,19 +1,13 @@ # Name -pfs_cp - copy files and directories +cp - copy files and directories # Synopsis -` pfs_cp +` cp [OPTION]... or or ` # Description -``` - --only-show-errors (boolean) - Only errors and warnings are displayed. All other output is suppressed. - - --page-size (integer) - The number of results to return in each response to a list operation. The default value is 1000 (the maximum allowed). Using a lower value may help if an operation times out. - +``` --preserve--links Reserve links when copy files @@ -22,10 +16,10 @@ pfs_cp - copy files and directories ``` # Examples -- The following command cp a single file to pfs +- The following command copies a single file to pfs ``` -pfs_cp ./text1.txt pfs://mydir/text1.txt +paddle pfs cp ./text1.txt pfs://mydir/text1.txt ``` Output @@ -34,10 +28,10 @@ Output upload ./text1.txt to pfs://mydir/text1.txt ``` -- The following command cp pfs file to a local file +- The following command copies pfs file to a local file ``` -pfs_cp pfs://mydir/text1.txt ./text1.txt +paddle pfs cp pfs://mydir/text1.txt ./text1.txt ``` Output diff --git a/doc/design/file_mananger/pfs/ls.md b/doc/design/file_mananger/pfs/ls.md index 41cf6fdf77f21138ba2ccc5c3b81f920e35af3a4..bec4c39b37eed2b0e9852cd4ab0ef127b469162b 100644 --- a/doc/design/file_mananger/pfs/ls.md +++ b/doc/design/file_mananger/pfs/ls.md @@ -1,49 +1,42 @@ # Name -pfs_ls - list directory contents o +ls - list directory contents or a file attributes # Synopsis -` pfs_ls [OPTION]... ` +` ls [OPTION]... ` # Description -``` - -h, --help - Display this help and exit - - --version - Output version information and exit - - --only-show-errors (boolean) - Only errors and warnings are displayed. All other output is suppressed. - - --page-size (integer) - The number of results to return in each response to a list operation. The default value is 1000 (the maximum allowed). Using a lower value may help if an operation times out. - +``` -R, -r, --recursive Copy directories recursively + + --page-size (integer) + The number of results to return in each response to a list operation. The default value is 1000 (the maximum allowed). Using a lower value may help if operation time out. ``` # Examples -- The following command cp a single file to pfs +- The following command lists a single file ``` -pfs_cp ./text1.txt pfs://mydir/text1.txt +paddle pfs ls pfs://mydir/text1.txt ``` Output ``` -upload ./text1.txt to pfs://mydir/text1.txt +2017-05-5 17:04:30 text1.txt ``` -- The following command cp pfs file to a local file +- The following command lists directory contents ``` -pfs_cp pfs://mydir/text1.txt ./text1.txt +paddle pfs ls pfs://mydir ``` Output ``` -download pfs://mydir/text1.txt to ./text1.txt +2017-05-5 17:04:30 text1.txt +2017-05-5 17:04:30 text2.txt +... ``` diff --git a/doc/design/file_mananger/pfs/mkdir.md b/doc/design/file_mananger/pfs/mkdir.md index e3b0d0cae78a6d1897da98f2f91dd2b93fb278b0..45906dc208380bb87b3e2bfddb2d38a47b8fe8ed 100644 --- a/doc/design/file_mananger/pfs/mkdir.md +++ b/doc/design/file_mananger/pfs/mkdir.md @@ -1,29 +1,15 @@ # Name -pdf_mkdir - mkdir directory(ies) +mkdir - mkdir directory(ies) # Synopsis -`pfs_mkdir [OPTION]... PFS_DIRECTORY...` +`mkdir [OPTION]... ...` # Description -Create the pfs directory(ies), if they do not already exist. - -``` - -h, --help - display this help and exit - - --version - output version information and exit - - --only-show-errors (boolean) - Only errors and warnings are displayed. All other output is suppressed. - - --page-size (integer) - The number of results to return in each response to a list operation. The default value is 1000 (the maximum allowed). Using a lower value may help if an operation times out. -``` +Create the pfs directory(ies), if they do not already exist. And create intermediate directories as required # Examples ``` -pfs_mkdir pfs://mydir1 +paddle pfs mkdir pfs://mydir1 ``` Output diff --git a/doc/design/file_mananger/pfs/mv.md b/doc/design/file_mananger/pfs/mv.md index 3f4c1ac0761dae4db9c007416bd2ca1c353a5f7e..7f7ef209c823e2febda30786c26b4f40468b0106 100644 --- a/doc/design/file_mananger/pfs/mv.md +++ b/doc/design/file_mananger/pfs/mv.md @@ -3,7 +3,8 @@ mv - move (rename) files or directories # Synopsis -If destination already exist, please rm it first. +If destination already exist, please [rm](rm.md) it first. + ``` mv [OPTION]... or or @@ -11,28 +12,17 @@ mv [OPTION]... # Description ``` - -h, --help - display this help and exit - - -v, --version - output version information and exit - - --only-show-errors (boolean) - Only errors and warnings are displayed. All other output is suppressed. - - --page-size (integer) - The number of results to return in each response to a list operation. The default value is 1000 (the maximum allowed). Using a lower value may help if an operation times out. ``` # Examples - The following command move a single file to pfs ``` -mv ./text1.txt pfs://mydirectory/test1.txt +paddle pfs mv ./text1.txt pfs://mydirectory/test1.txt ``` Output ``` -move text1.txt pfs://mydirectory/test1.txt +move ./text1.txt pfs://mydirectory/test1.txt ``` diff --git a/doc/design/file_mananger/pfs/pfs.md b/doc/design/file_mananger/pfs/pfs.md index a17de7d8140fe6cd42f957b9c4f21a22973a538f..76ee0a6917d133b93521e82d346acc1726ec40c5 100644 --- a/doc/design/file_mananger/pfs/pfs.md +++ b/doc/design/file_mananger/pfs/pfs.md @@ -12,20 +12,26 @@ paddle [options] pfs [parameters] ``` --profile (string) Use a specific profile from your credential file. ---help pfs - Display more information about pfs + +--help (string) + Display more information about command + --version Output version information and exit + --debug Show detailed debugging log + +--only-show-errors (boolean) + Only errors and warnings are displayed. All other output is suppressed. ``` ## Path Arguments -When using a commnd, we need to specify path arguments. There are two path argument type: `localpath` and `pfspath`. -A pfspath begin with `pfs://`, eg: `pfs://mydir/text1.txt`. +When using a command, we need to specify path arguments. There are two path argument type: `localpath` and `pfspath`. +A `pfspath` begin with `pfs://`, eg: `pfs://mydir/text1.txt`. -## Path Arguments‘s order -Commonly, there maybe two path arguments. The first is source, and the second is destination. +## order of Path Arguments +Commonly, if there are two path arguments. The first is the source, and the second is the destination. ## Subcommonds - [rm](rm.md) diff --git a/doc/design/file_mananger/pfs/rm.md b/doc/design/file_mananger/pfs/rm.md index 6056c401f956c040165becdcbc34709a74ae13b7..fa881fe1e94b3057b3276226e8eda3222b454528 100644 --- a/doc/design/file_mananger/pfs/rm.md +++ b/doc/design/file_mananger/pfs/rm.md @@ -2,30 +2,25 @@ rm - remove files or directories # Synopsis -`rm [OPTION]... PFS_DIRECTORY...` +``` +rm [OPTION]... ... +``` # Description -Create the directory, if it does not already exits ``` -r, -R, --recursive remove directories and their contents recursively - - -v, --version - output version information and exit - - --only-show-errors (boolean) - Only errors and warnings are displayed. All other output is suppressed. --page-size (integer) - The number of results to return in each response to a list operation. The default value is 1000 (the maximum allowed). Using a lower value may help if an operation times out. + The number of results to return in each response to a list operation. The default value is 1000 (the maximum allowed). Using a lower value may help if an operation times out ``` # Examples -- The following command delete a single file +- The following command deletes a single files ``` -pfs_rm pfs://mydirectory/test1.txt +paddle pfs rm pfs://mydirectory/test1.txt ``` Output @@ -35,10 +30,10 @@ delete pfs://mydirectory/test1.txt ``` -- The following command delete a directory recursively. +- The following command deletes a directory recursively. ``` -pfs_rm -r pfs://mydirectory +paddle pfs rm -r pfs://mydirectory ``` Output diff --git a/doc/design/file_mananger/pfs/sync.md b/doc/design/file_mananger/pfs/sync.md index 99248082743559c5e51866079e3bc0b199895a8a..7a8e0b74cf196c10eb1e7878befd5514b284be4f 100644 --- a/doc/design/file_mananger/pfs/sync.md +++ b/doc/design/file_mananger/pfs/sync.md @@ -1,53 +1,42 @@ # Name -pfs_cp - copy files and directories +sync - sync directories. Recursively copies new and updated files from the source directory to the destination # Synopsis -` pfs_cp +` sync [OPTIONS]... or or ` # Description ``` - -h, --help - Display this help and exit - - --version - Output version information and exit - - --only-show-errors (boolean) - Only errors and warnings are displayed. All other output is suppressed. - - --page-size (integer) - The number of results to return in each response to a list operation. The default value is 1000 (the maximum allowed). Using a lower value may help if an operation times out. - - --preserve--links - Reserve links when copy files - - -R, -r, --recursive - Copy directories recursively + -l, --links + copy symlinks as symlinks ``` # Examples -- The following command cp a single file to pfs +- The following command sync locally directory to pfs ``` -pfs_cp ./text1.txt pfs://mydir/text1.txt +sync ./dir1 pfs://mydir1 ``` Output ``` -upload ./text1.txt to pfs://mydir/text1.txt +upload ./dir1/text1.txt to pfs://mydir1/text2.txt +upload ./dir1/text2.txt to pfs://mydir1/text2.txt +... ``` -- The following command cp pfs file to a local file +- The following command sync pfs directory to local ``` -pfs_cp pfs://mydir/text1.txt ./text1.txt +sync pfs://mydir1 . ``` Output ``` -download pfs://mydir/text1.txt to ./text1.txt +download pfs://mydir1/text1.txt to ./text1.txt +download pfs://mydir1/text2.txt to ./text2.txt +... ```