未验证 提交 c7d60ce4 编写于 作者: Z zmxdream 提交者: GitHub

[GPUPS]add afs OpenWriter (#46611)

* add afs OpenWriter

* update
上级 1e2af54c
...@@ -57,7 +57,7 @@ limitations under the License. */ ...@@ -57,7 +57,7 @@ limitations under the License. */
#include "paddle/fluid/distributed/the_one_ps.pb.h" #include "paddle/fluid/distributed/the_one_ps.pb.h"
#endif #endif
#ifdef PADDLE_WITH_PSLIB #ifdef PADDLE_WITH_PSLIB
#include "afs_api.h" #include "afs_api.h" // NOLINT
#endif #endif
#ifdef PADDLE_WITH_PSLIB #ifdef PADDLE_WITH_PSLIB
#include "downpour_accessor.h" // NOLINT #include "downpour_accessor.h" // NOLINT
...@@ -504,7 +504,7 @@ class PSGPUWrapper { ...@@ -504,7 +504,7 @@ class PSGPUWrapper {
// set optimizer type(naive,adagrad,std_adagrad,adam,share_adam) // set optimizer type(naive,adagrad,std_adagrad,adam,share_adam)
optimizer_type_ = (config.find("optimizer_type") == config.end()) optimizer_type_ = (config.find("optimizer_type") == config.end())
? 1 ? 1
: int(config["optimizer_type"]); : static_cast<int>(config["optimizer_type"]);
VLOG(0) << "InitializeGPUServer optimizer_type_:" << optimizer_type_ VLOG(0) << "InitializeGPUServer optimizer_type_:" << optimizer_type_
<< " nodeid_slot:" << nodeid_slot << " nodeid_slot:" << nodeid_slot
...@@ -556,7 +556,7 @@ class PSGPUWrapper { ...@@ -556,7 +556,7 @@ class PSGPUWrapper {
if (slot_info_initialized_) { if (slot_info_initialized_) {
return; return;
} }
SlotRecordDataset* dataset = (SlotRecordDataset*)(dataset_); SlotRecordDataset* dataset = reinterpret_cast<SlotRecordDataset*>(dataset_);
auto slots_vec = dataset->GetSlots(); auto slots_vec = dataset->GetSlots();
slot_offset_vector_.clear(); slot_offset_vector_.clear();
for (auto& slot : slot_vector_) { for (auto& slot : slot_vector_) {
...@@ -620,6 +620,11 @@ class PSGPUWrapper { ...@@ -620,6 +620,11 @@ class PSGPUWrapper {
return afs_handler_.open_reader(filename); return afs_handler_.open_reader(filename);
} }
std::shared_ptr<paddle::ps::AfsWriter> OpenWriter(
const std::string& filename) {
return afs_handler_.open_writer(filename);
}
void InitAfsApi(const std::string& fs_name, void InitAfsApi(const std::string& fs_name,
const std::string& fs_user, const std::string& fs_user,
const std::string& pass_wd, const std::string& pass_wd,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册