diff --git a/paddle/fluid/framework/fleet/ps_gpu_wrapper.cc b/paddle/fluid/framework/fleet/ps_gpu_wrapper.cc index 75f5c24af5a9961bafbc7296299c55bd46bce3ef..c7852de00a18ef08d6fcbb3ff85e9b003e9834c1 100755 --- a/paddle/fluid/framework/fleet/ps_gpu_wrapper.cc +++ b/paddle/fluid/framework/fleet/ps_gpu_wrapper.cc @@ -37,6 +37,43 @@ limitations under the License. */ namespace paddle { namespace framework { +#ifdef PADDLE_WITH_PSLIB +void AfsWrapper::init(const std::string& fs_name, const std::string& fs_user, + const std::string& pass_wd, const std::string& conf) { + int ret = afs_handler_.init(fs_name.c_str(), fs_user.c_str(), pass_wd.c_str(), + conf.c_str()); + if (ret != 0) { + LOG(ERROR) << "AFS Init Error"; + } +} + +int AfsWrapper::remove(const std::string& path) { + return afs_handler_.remove(path); +} + +int AfsWrapper::mkdir(const std::string& path) { + return afs_handler_.mkdir(path); +} + +std::vector AfsWrapper::list(const std::string& path) { + return afs_handler_.list(path); +} + +int AfsWrapper::exist(const std::string& path) { + return afs_handler_.exist(path); +} + +int AfsWrapper::upload(const std::string& local_file, + const std::string& afs_file) { + return afs_handler_.upload_file(local_file, afs_file); +} + +int AfsWrapper::download(const std::string& local_file, + const std::string& afs_file) { + return afs_handler_.download_file(local_file, afs_file); +} +#endif + std::shared_ptr PSGPUWrapper::s_instance_ = NULL; bool PSGPUWrapper::is_initialized_ = false; #ifdef PADDLE_WITH_PSLIB diff --git a/paddle/fluid/framework/fleet/ps_gpu_wrapper.h b/paddle/fluid/framework/fleet/ps_gpu_wrapper.h index d9d29cc072dd7bc7beec62bedcd1d05ddac726ce..9b7d6de082d1c1c6e2b3d16f91522304524df99a 100755 --- a/paddle/fluid/framework/fleet/ps_gpu_wrapper.h +++ b/paddle/fluid/framework/fleet/ps_gpu_wrapper.h @@ -55,6 +55,27 @@ namespace framework { #define TYPEALIGN(ALIGNVAL, LEN) \ (((uint64_t)(LEN) + ((ALIGNVAL)-1)) & ~((uint64_t)((ALIGNVAL)-1))) +#ifdef PADDLE_WITH_PSLIB +class AfsWrapper { + public: + AfsWrapper() {} + virtual ~AfsWrapper() {} + void init(const std::string& fs_name, const std::string& fs_user, + const std::string& pass_wd, const std::string& conf); + int remove(const std::string& path); + int mkdir(const std::string& path); + std::vector list(const std::string& path); + + int exist(const std::string& path); + int upload(const std::string& local_file, const std::string& afs_file); + + int download(const std::string& local_file, const std::string& afs_file); + + private: + paddle::ps::AfsApiWrapper afs_handler_; +}; +#endif + class PSGPUWrapper { public: virtual ~PSGPUWrapper() { delete HeterPs_; } diff --git a/paddle/fluid/pybind/ps_gpu_wrapper_py.cc b/paddle/fluid/pybind/ps_gpu_wrapper_py.cc index fe1f27226bad4b95bb85e7f67dbd940a2058b92a..79529fca7d1be9a19cd02c080ea61f034156528b 100644 --- a/paddle/fluid/pybind/ps_gpu_wrapper_py.cc +++ b/paddle/fluid/pybind/ps_gpu_wrapper_py.cc @@ -63,6 +63,27 @@ void BindPSGPUWrapper(py::module* m) { .def("finalize", &framework::PSGPUWrapper::Finalize, py::call_guard()); } // end PSGPUWrapper +#ifdef PADDLE_WITH_PSLIB +void BindAfsWrapper(py::module* m) { + py::class_>( + *m, "AfsWrapper") + .def(py::init([]() { return std::make_shared(); })) + .def("init", &framework::AfsWrapper::init, + py::call_guard()) + .def("list", &framework::AfsWrapper::list, + py::call_guard()) + .def("mkdir", &framework::AfsWrapper::mkdir, + py::call_guard()) + .def("exist", &framework::AfsWrapper::exist, + py::call_guard()) + .def("download", &framework::AfsWrapper::download, + py::call_guard()) + .def("upload", &framework::AfsWrapper::upload, + py::call_guard()) + .def("remove", &framework::AfsWrapper::remove, + py::call_guard()); +} +#endif #endif } // end namespace pybind } // end namespace paddle diff --git a/paddle/fluid/pybind/ps_gpu_wrapper_py.h b/paddle/fluid/pybind/ps_gpu_wrapper_py.h index ba4f146389ed3ef855336c9af10ccd857886e176..22cd5ef0fd1496c9da518512e49904d8f10256dc 100644 --- a/paddle/fluid/pybind/ps_gpu_wrapper_py.h +++ b/paddle/fluid/pybind/ps_gpu_wrapper_py.h @@ -24,6 +24,9 @@ namespace pybind { #ifdef PADDLE_WITH_HETERPS void BindPSGPUWrapper(py::module* m); +#ifdef PADDLE_WITH_PSLIB +void BindAfsWrapper(py::module* m); +#endif #endif } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 44abf3357d63d0a32722eaae67648c0c2ad2cf55..c9e304e696df287f7d846a6fc463cf746a24c738 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -4458,6 +4458,9 @@ All parameter, weight, gradient are variables in Paddle. #endif #ifdef PADDLE_WITH_HETERPS BindPSGPUWrapper(&m); +#ifdef PADDLE_WITH_PSLIB + BindAfsWrapper(&m); +#endif #endif BindGlooWrapper(&m); BindBoxHelper(&m);