From db3cc7a78c17cb804c1c4bc9baa35411fb7f3b37 Mon Sep 17 00:00:00 2001 From: rensilin Date: Wed, 31 Jul 2019 21:44:49 +0800 Subject: [PATCH] finish execute Change-Id: I76a7a42deaab3f748d958d03c26d88e808f6c6d8 --- BCLOUD | 2 + .../custom_trainer/feed/executor/executor.cc | 61 ++++++++++++------- .../fluid/train/custom_trainer/feed/main.cc | 1 + .../custom_trainer/feed/unit_test/main.cc | 1 + .../feed/unit_test/test_executor.cc | 36 +++++++++++ 5 files changed, 79 insertions(+), 22 deletions(-) diff --git a/BCLOUD b/BCLOUD index 7b4b293a..28265b3c 100644 --- a/BCLOUD +++ b/BCLOUD @@ -84,6 +84,8 @@ def UT_FILE(filename): UT_DIR = 'paddle/fluid/train/custom_trainer/feed/unit_test' return os.path.join(UT_DIR, filename) +CPPFLAGS('-DPYBIND_AVX_MKLML') # for paddle with avx and mklml + custom_trainer_src = GLOB('paddle/fluid/train/custom_trainer/feed/*/*.cc', Exclude(UT_FILE('*'))) Application('feed_trainer', Sources('paddle/fluid/train/custom_trainer/feed/main.cc', custom_trainer_src), CppFlags('-DHPPL_STUB_FUNC -DLAPACK_FOUND -DPADDLE_DISABLE_PROFILER -DPADDLE_NO_PYTHON -DCUSTOM_TRAINER -DPADDLE_ON_INFERENCE -DPADDLE_USE_DSO -DPADDLE_USE_PTHREAD_BARRIER -DPADDLE_USE_PTHREAD_SPINLOCK -DPADDLE_VERSION=0.0.0 -DPADDLE_WITH_AVX -DPADDLE_WITH_MKLML -DPADDLE_WITH_XBYAK -DXBYAK64 -DXBYAK_NO_OP_NAMES -D_GNU_SOURCE -D__STDC_LIMIT_MACROS -DPYBIND_AVX_MKLML' + r" -DPADDLE_REVISION=\"%s@%s@%s\"" % (REPO_URL(), REPO_BRANCH(), REPO_REVISION())), CFlags('-std=c++11 -m64 -fPIC -fno-omit-frame-pointer -Werror -Wall -Wextra -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter -Wno-unused-function -Wno-error=literal-suffix -Wno-error=sign-compare -Wno-error=unused-local-typedefs -Wno-error=maybe-uninitialized -fopenmp -mavx -O3 -DNDEBUG '), CxxFlags('-std=c++11 -m64 -fPIC -fno-omit-frame-pointer -Werror -Wall -Wextra -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter -Wno-unused-function -Wno-error=literal-suffix -Wno-error=sign-compare -Wno-error=unused-local-typedefs -Wno-error=maybe-uninitialized -fopenmp -mavx -O3 -DNDEBUG '), Libraries('$OUT/lib/libpaddle_fluid_avx_mklml.a')) diff --git a/paddle/fluid/train/custom_trainer/feed/executor/executor.cc b/paddle/fluid/train/custom_trainer/feed/executor/executor.cc index b0d9a021..4f02b519 100644 --- a/paddle/fluid/train/custom_trainer/feed/executor/executor.cc +++ b/paddle/fluid/train/custom_trainer/feed/executor/executor.cc @@ -5,6 +5,7 @@ #include "paddle/fluid/platform/init.h" #include "paddle/fluid/platform/cpu_helper.h" #include "paddle/fluid/inference/api/details/reset_tensor_array.h" +#include "paddle/fluid/platform/enforce.h" namespace paddle { @@ -16,7 +17,7 @@ namespace { int ReadBinaryFile(const std::string& filename, std::string* contents) { std::ifstream fin(filename, std::ios::in | std::ios::binary); if (!fin) { - VLOG(4) << "Cannot open file " << filename; + VLOG(2) << "Cannot open file " << filename; return -1; } fin.seekg(0, std::ios::end); @@ -73,37 +74,53 @@ int SimpleExecute::initialize(YAML::Node& exe_config, if (!exe_config["startup_program"] || !exe_config["main_program"]) { - VLOG(4) << "fail to load config"; + VLOG(2) << "fail to load config"; return -1; } - _context.reset(new SimpleExecute::Context(context_ptr->cpu_place)); - auto startup_program = Load(&_context->executor, exe_config["startup_program"].as()); - if (startup_program == nullptr) { - VLOG(4) << "fail to load startup_program: " << exe_config["startup_program"].as(); + try { + _context.reset(new SimpleExecute::Context(context_ptr->cpu_place)); + auto startup_program = Load(&_context->executor, exe_config["startup_program"].as()); + if (startup_program == nullptr) { + VLOG(2) << "fail to load startup_program: " << exe_config["startup_program"].as(); + return -1; + } + + _context->executor.Run(*startup_program, this->scope(), 0, false, true); + + _context->main_program = Load(&_context->executor, exe_config["main_program"].as()); + if (_context->main_program == nullptr) { + VLOG(2) << "fail to load main_program: " << exe_config["main_program"].as(); + return -1; + } + _context->prepare_context = _context->executor.Prepare(*_context->main_program, 0); + _context->executor.CreateVariables(*_context->main_program, this->scope(), 0); + } catch (::paddle::platform::EnforceNotMet& err) { + VLOG(2) << err.what(); + _context.reset(nullptr); return -1; } - - _context->executor.Run(*startup_program, this->scope(), 0, false, true); - _context->main_program = Load(&_context->executor, exe_config["main_program"].as()); - if (_context->main_program == nullptr) { - VLOG(4) << "fail to load main_program: " << exe_config["main_program"].as(); - return -1; - } - _context->prepare_context = _context->executor.Prepare(*_context->main_program, 0); - _context->executor.CreateVariables(*_context->main_program, this->scope(), 0); return 0; } int SimpleExecute::run() { - _context->executor.RunPreparedContext(_context->prepare_context.get(), this->scope(), - false, /* don't create local scope each time*/ - false /* don't create variable each time */); - - // For some other vector like containers not cleaned after each batch. - _context->tensor_array_batch_cleaner.CollectNoTensorVars(this->scope()); - _context->tensor_array_batch_cleaner.ResetNoTensorVars(); + if (_context == nullptr) { + VLOG(2) << "need initialize before run"; + return -1; + } + try { + _context->executor.RunPreparedContext(_context->prepare_context.get(), this->scope(), + false, /* don't create local scope each time*/ + false /* don't create variable each time */); + + // For some other vector like containers not cleaned after each batch. + _context->tensor_array_batch_cleaner.CollectNoTensorVars(this->scope()); + _context->tensor_array_batch_cleaner.ResetNoTensorVars(); + } catch (::paddle::platform::EnforceNotMet& err) { + VLOG(2) << err.what(); + return -1; + } return 0; } diff --git a/paddle/fluid/train/custom_trainer/feed/main.cc b/paddle/fluid/train/custom_trainer/feed/main.cc index 51784119..6da66946 100644 --- a/paddle/fluid/train/custom_trainer/feed/main.cc +++ b/paddle/fluid/train/custom_trainer/feed/main.cc @@ -5,6 +5,7 @@ #include "paddle/fluid/train/custom_trainer/feed/trainer_context.h" #include "paddle/fluid/train/custom_trainer/feed/process/process.h" #include "paddle/fluid/train/custom_trainer/feed/process/init_env_process.h" +#include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/pybind/pybind.h" using namespace paddle::custom_trainer::feed; diff --git a/paddle/fluid/train/custom_trainer/feed/unit_test/main.cc b/paddle/fluid/train/custom_trainer/feed/unit_test/main.cc index 512189ba..7ff531d8 100644 --- a/paddle/fluid/train/custom_trainer/feed/unit_test/main.cc +++ b/paddle/fluid/train/custom_trainer/feed/unit_test/main.cc @@ -1,6 +1,7 @@ #include #include #include +#include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/pybind/pybind.h" int32_t main(int32_t argc, char** argv) { diff --git a/paddle/fluid/train/custom_trainer/feed/unit_test/test_executor.cc b/paddle/fluid/train/custom_trainer/feed/unit_test/test_executor.cc index eac7d286..ffecb3e8 100644 --- a/paddle/fluid/train/custom_trainer/feed/unit_test/test_executor.cc +++ b/paddle/fluid/train/custom_trainer/feed/unit_test/test_executor.cc @@ -12,9 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include #include #include "paddle/fluid/train/custom_trainer/feed/executor/executor.h" +#include "paddle/fluid/framework/tensor_util.h" namespace paddle { namespace custom_trainer { @@ -31,12 +33,46 @@ TEST(testSimpleExecute, initialize) { ASSERT_EQ(0, execute.initialize(config, context_ptr)); } +float uniform(float min, float max) { + float result = (float)rand() / RAND_MAX; + return min + result * (max - min); +} + +void next_batch(int batch_size, const paddle::platform::Place& place, paddle::framework::LoDTensor* x_tensor, paddle::framework::LoDTensor* y_tensor) { + + x_tensor->Resize({batch_size, 2}); + auto x_data = x_tensor->mutable_data(place); + + y_tensor->Resize({batch_size, 1}); + auto y_data = y_tensor->mutable_data(place); + + for (int i = 0; i < batch_size; ++i) { + x_data[i * 2] = uniform(-2, 2); + x_data[i * 2 + 1] = uniform(-2, 2); + float dis = x_data[i * 2] * x_data[i * 2] + x_data[i * 2 + 1] * x_data[i * 2 + 1]; + y_data[i] = dis < 1.0 ? 1.0 : 0.0; + } +} + TEST(testSimpleExecute, run) { SimpleExecute execute; auto context_ptr = std::make_shared(); auto config = YAML::Load("{thread_num: 2, startup_program: ./data/startup_program, main_program: ./data/main_program}"); ASSERT_EQ(0, execute.initialize(config, context_ptr)); + + + auto x_var = execute.mutable_var<::paddle::framework::LoDTensor>("x"); + auto y_var = execute.mutable_var<::paddle::framework::LoDTensor>("y"); + ASSERT_NE(nullptr, x_var); + ASSERT_NE(nullptr, y_var); + + next_batch(1024, context_ptr->cpu_place, x_var, y_var); + ASSERT_EQ(0, execute.run()); + + auto loss_var = execute.var<::paddle::framework::LoDTensor>("loss"); + auto loss = loss_var.data()[0]; + std::cout << "loss: " << loss << std::endl; } } // namespace feed -- GitLab