Trainer.cpp 22.3 KB
Newer Older
Z
zhangjinchao01 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "Trainer.h"

#include <fenv.h>
#include <stdio.h>

#include <iomanip>
Y
Yu Yang 已提交
21
#include <iostream>
Z
zhangjinchao01 已提交
22
#include <limits>
Y
Yu Yang 已提交
23
#include <sstream>
Z
zhangjinchao01 已提交
24 25 26

#include <google/protobuf/text_format.h>

Y
Yu Yang 已提交
27 28
#include "paddle/utils/Excepts.h"
#include "paddle/utils/GlobalConstants.h"
Z
zhangjinchao01 已提交
29 30 31 32
#include "paddle/utils/PythonUtil.h"
#include "paddle/utils/Stat.h"
#include "paddle/utils/Util.h"

Y
Yu Yang 已提交
33
#include "RemoteParameterUpdater.h"
Z
zhangjinchao01 已提交
34 35 36
#include "TesterConfig.h"
#include "ThreadParameterUpdater.h"
#include "TrainerConfigHelper.h"
Y
Yu Yang 已提交
37 38 39
#include "paddle/gserver/gradientmachines/GradientMachineMode.h"
#include "paddle/gserver/gradientmachines/NeuralNetwork.h"
#include "paddle/gserver/layers/ValidationLayer.h"
Z
zhangjinchao01 已提交
40 41 42

P_DEFINE_string(config, "", "Trainer config file");

Y
Yu Yang 已提交
43 44
P_DEFINE_int32(test_period,
               0,
W
wangyanfei01 已提交
45
               "if equal 0, do test on all test data at the end of "
W
wangyanfei01 已提交
46 47
               "each pass. While if equal non-zero, do test on all test "
               "data every test_period batches");
Y
Yu Yang 已提交
48 49 50 51
P_DEFINE_bool(test_all_data_in_one_period,
              false,
              "This option was deprecated, since we will always do "
              "test on all test set ");
52

Z
zhangjinchao01 已提交
53 54
P_DEFINE_bool(local, true, "Train in local mode or not");

55 56
P_DEFINE_int32(average_test_period,
               0,
Z
zhangjinchao01 已提交
57 58 59 60 61
               "Do test on average parameter every so"
               " many batches. MUST be devided by FLAGS_log_period."
               " Default 0 means do not test average parameter");

P_DEFINE_int32(saving_period, 1, "Save parameteres every so many passes");
62 63
P_DEFINE_int64(saving_period_by_batches,
               0,
Z
zhangjinchao01 已提交
64 65
               "Save parameters every so many batches in one pass");
P_DEFINE_string(save_dir, "", "Directory for saving model parameter");
66 67
P_DEFINE_int32(start_pass,
               0,
Z
zhangjinchao01 已提交
68 69
               "Start training from this pass. "
               "Will load parameter from the previous pass");
70 71
P_DEFINE_int32(test_pass,
               -1,
Z
zhangjinchao01 已提交
72 73 74 75 76 77 78
               "Will load parameter start from this pass to test");
P_DEFINE_int32(test_wait, 0, "Waiting for pass parameter if not exist");
P_DEFINE_bool(with_cost, true, "enable cost layer or not");
P_DEFINE_bool(distribute_test, false, "test in distribute mode");

P_DEFINE_int32(num_passes, 100, "train for so many passes");

79 80
P_DEFINE_string(config_args,
                "",
Z
zhangjinchao01 已提交
81 82 83
                "arguments passed to config file."
                "Format: key1=value1,key2=value2");

84 85
P_DEFINE_bool(save_only_one,
              false,
Z
zhangjinchao01 已提交
86 87 88
              "Save only parameters in last pass, remove previous.");

P_DEFINE_string(feat_file, "", "File name of extracted feature.");
89 90
P_DEFINE_string(predict_output_dir,
                "",
Z
zhangjinchao01 已提交
91
                "Directory that saves the predicted results of output layers");
92 93
P_DEFINE_string(model_list,
                "",
Z
zhangjinchao01 已提交
94 95 96 97 98 99 100 101 102 103 104 105 106 107
                "File that saves the model list when evaluation");

namespace paddle {

void Trainer::init(int argc, char** argv) {
  initMain(argc, argv);
  initPython(argc, argv);

  auto config = TrainerConfigHelper::createFromFlagConfig();
  feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW);

  init(config);
}

108
void Trainer::init(const std::shared_ptr<TrainerConfigHelper>& config,
Z
zhangjinchao01 已提交
109
                   bool testing,
110 111 112
                   const std::shared_ptr<GradientMachine>& gradientMachine,
                   const std::shared_ptr<DataProvider>& dataProvider,
                   const std::shared_ptr<DataProvider>& testDataProvider) {
Z
zhangjinchao01 已提交
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
  this->stats_ = std::make_shared<TrainerStats>();

  config_ = config;

  config_->updateConfigFromFlags();

  testing_ = testing;

  // in testing, mode_ may GradientMachine::kTesting or
  // GradientMachine::kSgdSparseCpuTraining

  if (FLAGS_local) {
    CHECK(!FLAGS_loadsave_parameters_in_pserver)
        << "local and loadsave_parameters_in_pserver can not both true";
    if (config_->getOptConfig().use_sparse_remote_updater()) {
      config_->disableRemoteSparseUpdaterForEachParams();
      LOG(INFO) << "ignore sparse_remote_update=true due to  --local=true";
    }
  }
  if (FLAGS_loadsave_parameters_in_pserver) {
    CHECK(config_->getOptConfig().use_sparse_remote_updater())
        << "no parameter to load from pserver, please check network config";
  }
  if (testing && !FLAGS_loadsave_parameters_in_pserver) {
    if (config_->getOptConfig().use_sparse_remote_updater()) {
      config_->disableRemoteSparseUpdater();
      LOG(INFO) << "because parameter is loaded local,"
                << "tester ignore sparse_remote_update flag";
    }
  }

  CHECK(TrainAlgorithm::isValid(config_->getOptConfig().algorithm()))
      << "invalid algorithm configuration: "
      << config_->getOptConfig().algorithm();

  bool useSparseUpdater = false;
  for (auto& paraConfig : config_->getModelConfig().parameters()) {
    if (paraConfig.sparse_update() || paraConfig.sparse_remote_update()) {
      useSparseUpdater = true;
    }
  }

  if (testing) {
    LOG(INFO) << "trainer: in testing mode";
    if (config_->getOptConfig().use_sparse_remote_updater() ||
        FLAGS_trainer_count > 1) {
      mode_ = GradientMachine::kSgdSparseCpuTraining;
      LOG(INFO) << "trainer mode: SgdSparseCpuTraining";
    } else {
      mode_ = GradientMachine::kTesting;
      LOG(INFO) << "trainer mode: Testing";
    }
  } else if (IGradientMachineMode::tryGetMode(
166 167 168 169 170
                 (int*)&mode_,
                 config_->getOptConfig().algorithm(),
                 FLAGS_trainer_count,
                 FLAGS_local,
                 FLAGS_use_gpu)) {
Z
zhangjinchao01 已提交
171 172
    LOG(INFO) << "Custom trainer mode.";
  } else if ((config_->getOptConfig().algorithm() == TrainAlgorithm::SGD ||
173 174 175
              config_->getOptConfig().algorithm() ==
                  TrainAlgorithm::AsyncSGD) &&
             useSparseUpdater) {
Z
zhangjinchao01 已提交
176 177 178 179 180 181 182 183
    mode_ = GradientMachine::kSgdSparseCpuTraining;
    LOG(INFO) << "trainer mode: SgdSparseCpuTraining";
  } else {
    mode_ = GradientMachine::kNormal;
    LOG(INFO) << "trainer mode: Normal";
  }

  // initialize trainer internal
184 185
  trainerInternal_.init(config_,
                        gradientMachine,
Z
zhangjinchao01 已提交
186
                        TrainerInternalConfig::createFromMode(mode_),
187 188
                        stats_,
                        testing);
Z
zhangjinchao01 已提交
189
  std::unique_ptr<ParameterUtilConfig> paramConfig(
190 191 192 193
      new ParameterUtilConfig(FLAGS_save_only_one,
                              FLAGS_saving_period,
                              FLAGS_loadsave_parameters_in_pserver,
                              FLAGS_config));
Z
zhangjinchao01 已提交
194 195

  paramUtil_.reset(
196 197 198 199
      new paddle::ParameterUtil(config_,
                                std::move(paramConfig),
                                trainerInternal_.getGradientMachine(),
                                trainerInternal_.getParameterUpdater()));
Z
zhangjinchao01 已提交
200

201 202 203
  bool gpuData =
      FLAGS_use_gpu && (!FLAGS_parallel_nn) &&
      (!IGradientMachineMode::dataMustInCpu(mode_, FLAGS_trainer_count));
Z
zhangjinchao01 已提交
204 205

  dataProvider_ = dataProvider;
X
xuwei06 已提交
206
  if (!dataProvider_ && config_->hasDataConfig() && !testing_) {
207
    dataProvider_.reset(DataProvider::create(*config_, *config_, gpuData));
Z
zhangjinchao01 已提交
208
  }
E
emailweixu 已提交
209 210
  if (!testDataProvider_) {
    // No evaluator_ if there is testDataProvider but no dataProvider.
Z
zhangjinchao01 已提交
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
    evaluator_.reset(trainerInternal_.getGradientMachine()->makeEvaluator());
    currentEvaluator_.reset(
        trainerInternal_.getGradientMachine()->makeEvaluator());
    if (FLAGS_average_test_period > 0 && FLAGS_trainer_id == 0 &&
        config_->getOptConfig().average_window() > 0) {
      CHECK_EQ(FLAGS_average_test_period % FLAGS_log_period, 0)
          << "FLAGS_average_test_period must be divided by FALGS_log_period";
      averageEvaluator_.reset(
          trainerInternal_.getGradientMachine()->makeEvaluator());
    }
  }

  testDataProvider_ = testDataProvider;
  if (!testDataProvider_ && config_->hasTestDataConfig()) {
    testDataProvider_.reset(
226
        DataProvider::create(config_->getTestDataConfig(), *config_, gpuData));
Z
zhangjinchao01 已提交
227 228
  }
  if (testDataProvider_) {
E
emailweixu 已提交
229
    createTester();
Z
zhangjinchao01 已提交
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
  }

  if (!testing &&
      (trainerInternal_.getGradientMachine()->hasStaticParameters())) {
    CHECK(!FLAGS_loadsave_parameters_in_pserver)
        << "is_static and loadsave_parameters_in_pserver can not both true";
  }
  if (testing) {
    // will load per pass for tester
  } else if (paramUtil_->tryLoadParametersFromConfig()) {
    // load from config already.
  } else {
    trainerInternal_.getGradientMachine()->randParameters();
  }

  // Only non static parameters need to be updated
  std::vector<ParameterPtr>& parameters =
      trainerInternal_.getGradientMachine()->getNonStaticParameters();
  if (trainerInternal_.getParameterUpdater()) {
    trainerInternal_.getParameterUpdater()->init(parameters);

    if (FLAGS_loadsave_parameters_in_pserver && FLAGS_trainer_id == 0) {
      if (testing) {
        // will load per pass for tester
      } else if (!config_->getConfig().init_model_path().empty() &&
                 (FLAGS_local || FLAGS_trainer_id == 0)) {
        paramUtil_->loadParametersWithPath(
257 258 259
            config_->getConfig().init_model_path(),
            false /*local*/,
            true /*remote*/);
Z
zhangjinchao01 已提交
260 261 262
      } else if (config_->getConfig().start_pass() > 0 &&
                 (FLAGS_local || FLAGS_trainer_id == 0)) {
        CHECK(paramUtil_->loadParameters(config_->getConfig().start_pass() - 1,
263 264
                                         false /*local*/,
                                         true /*remote*/));
Z
zhangjinchao01 已提交
265 266 267 268 269 270 271 272 273 274 275 276
      } else {
        trainerInternal_.getParameterUpdater()->randParametersRemote();
      }
    }
  }

  // set current evaluator and evalutor
  trainerInternal_.setCurrentEvaluator(currentEvaluator_.get());
  trainerInternal_.setEvaluator(evaluator_.get());
}

void Trainer::train(size_t numPasses) {
E
emailweixu 已提交
277
  startTrain();
Z
zhangjinchao01 已提交
278 279 280 281
  for (size_t i = 0; i < numPasses; ++i) {
    if (IGradientMachineMode::trainWholeDataInOneBatch(mode_)) {
      trainOnePassBatch(config_->getConfig().start_pass() + i);
    } else {
E
emailweixu 已提交
282
      trainOnePass();
Z
zhangjinchao01 已提交
283 284 285 286 287 288
    }
    if (i < numPasses - 1) {
      dataProvider_->reset();
    }
  }

E
emailweixu 已提交
289
  finishTrain();
Z
zhangjinchao01 已提交
290 291 292
}

static double genPerturbation(real* d, real* grad, size_t dim) {
293
  auto& reng = ThreadLocalRandomEngine::get();
Z
zhangjinchao01 已提交
294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
  std::uniform_real_distribution<double> dist(-1, 1);
  double gradNorm = 0, dNorm = 0;
  for (size_t i = 0; i < dim; ++i) {
    d[i] = dist(reng);
    dNorm += d[i] * d[i];
    gradNorm += grad[i] * grad[i];
  }
  if (gradNorm > 0) {
    real s = 0.5 * sqrt(gradNorm / dNorm);
    for (size_t i = 0; i < dim; ++i) {
      d[i] = s * d[i] + grad[i];
    }
  }
  double delta = 0;
  for (size_t i = 0; i < dim; ++i) {
    delta += grad[i] * d[i];
  }
  return delta;
}

real Trainer::checkGradient() {
  trainerInternal_.getGradientMachine()->start(*config_, dataProvider_);
  std::vector<ParameterPtr>& parameters =
      trainerInternal_.getGradientMachine()->getNonStaticParameters();
  DataBatch dataBatch;
  int32_t batchSize = config_->getOptConfig().batch_size();

  dataProvider_->getNextBatch(batchSize, &dataBatch);

  CHECK(dataBatch.getSize()) << "No data from data provider";
  std::vector<Argument>& inArgs = dataBatch.getStreams();
  std::vector<Argument> outArgs;

  trainerInternal_.getGradientMachine()->forward(inArgs, &outArgs, PASS_GC);
  real cost = Argument::sumCosts(outArgs);
  LOG(INFO) << "original cost=" << cost;
  trainerInternal_.getGradientMachine()->backward();

  real maxDiff = 0;
  char fill = ' ';
  for (auto& parameter : parameters) {
    CpuVector oldPara(parameter->getSize());
    CpuVector newPara(parameter->getSize());
    oldPara.copyFrom(*parameter->getBuf(PARAMETER_VALUE));
    real* newp = newPara.getData();
    real* oldp = oldPara.getData();
    CpuVector cpuGrad(*parameter->getBuf(PARAMETER_GRADIENT));
    real* grad = cpuGrad.getData();
    size_t dim = parameter->getSize();
    std::vector<real> d(dim);

    double delta = genPerturbation(d.data(), grad, dim);

    // use a step such that delta / cost is FLAGS_checkgrad_eps
    real step =
        (delta != 0) ? cost / delta * FLAGS_checkgrad_eps : FLAGS_checkgrad_eps;
    delta *= step;
    for (size_t i = 0; i < dim; ++i) {
      newp[i] = oldp[i] + step * d[i];
    }

    parameter->getBuf(PARAMETER_VALUE)->copyFrom(newPara);
    parameter->setValueUpdated();
    trainerInternal_.getGradientMachine()->forward(inArgs, &outArgs, PASS_GC);
    real newCost1 = Argument::sumCosts(outArgs);

    for (size_t i = 0; i < dim; ++i) {
      newp[i] = oldp[i] - step * d[i];
    }

    parameter->getBuf(PARAMETER_VALUE)->copyFrom(newPara);
    parameter->setValueUpdated();
    trainerInternal_.getGradientMachine()->forward(inArgs, &outArgs, PASS_GC);
    real newCost2 = Argument::sumCosts(outArgs);

    real trueDelta = 0.5 * (newCost1 - newCost2);
    real diff = (1e-20 + trueDelta) / (1e-20 + delta) - 1;
    LOG(INFO) << std::setiosflags(std::ios::left) << std::setfill(fill)
              << std::setw(20) << parameter->getName()
              << "step=" << std::setw(15) << step << "cost1=" << std::setw(10)
              << newCost1 << "cost2=" << std::setw(10) << newCost2
              << "true_delta=" << std::setw(15) << trueDelta
              << "analytic_delta=" << std::setw(15) << delta << "diff=" << diff
              << (std::abs(diff) > 0.01 ? " ***" : "");

    maxDiff = std::max(maxDiff, std::abs(diff));

    // restore parameter
    parameter->getBuf(PARAMETER_VALUE)->copyFrom(oldPara);
    parameter->setValueUpdated();

    fill = (fill == ' ') ? '.' : ' ';
  }
  return maxDiff;
}

E
emailweixu 已提交
390 391 392 393 394 395 396 397 398 399
void Trainer::startTrain() {
  trainPassContext_.passId = config_->getConfig().start_pass();
  srand(config_->getConfig().start_pass() + 1);
  if (dataProvider_) {
    dataProvider_->reset();
  }

  trainerInternal_.getGradientMachine()->start(*config_, dataProvider_);
}

400
void Trainer::finishTrain() { trainerInternal_.getGradientMachine()->finish(); }
E
emailweixu 已提交
401 402 403 404 405 406 407

void Trainer::startTrainPass() {
  stats_->reset();
  trainPassContext_.batchId = 0;
  trainPassContext_.avgTestCost = 0;
  trainPassContext_.numAvgTests = 0;
  trainPassContext_.passInnerId = 1;
Z
zhangjinchao01 已提交
408 409 410 411 412 413 414

  trainerInternal_.getParameterUpdater()->startPass();
  evaluator_->start();
  if (FLAGS_prev_batch_state) {
    trainerInternal_.getGradientMachine()->resetState();
    trainerInternal_.getGradientMachine()->getState(testState_);
  }
E
emailweixu 已提交
415
}
Z
zhangjinchao01 已提交
416

E
emailweixu 已提交
417 418 419 420 421 422 423
void Trainer::trainOneDataBatch(DataBatch& dataBatch) {
  int num = dataBatch.getSize();
  if (averageEvaluator_) {
    int64_t mod = trainPassContext_.batchId % FLAGS_average_test_period;
    if (mod >= FLAGS_average_test_period - FLAGS_log_period) {
      if (mod == FLAGS_average_test_period - FLAGS_log_period) {
        averageEvaluator_->start();
Z
zhangjinchao01 已提交
424
      }
E
emailweixu 已提交
425 426 427 428
      trainerInternal_.getParameterUpdater()->apply();
      if (FLAGS_prev_batch_state) {
        trainerInternal_.getGradientMachine()->getState(trainState_);
      }
429 430
      trainPassContext_.avgTestCost += tester_->forwardOneBatch(
          dataBatch, averageEvaluator_.get(), &forwardOutput_);
E
emailweixu 已提交
431 432 433 434 435
      if (FLAGS_prev_batch_state) {
        trainerInternal_.getGradientMachine()->setState(trainState_);
      }
      trainPassContext_.numAvgTests += num;
      trainerInternal_.getParameterUpdater()->restore();
Z
zhangjinchao01 已提交
436
    }
E
emailweixu 已提交
437 438 439 440
  }
  {
    REGISTER_TIMER("TrainBatch");
    trainerInternal_.trainOneBatch(
441
        trainPassContext_.batchId, dataBatch, &forwardOutput_);
E
emailweixu 已提交
442
  }
Z
zhangjinchao01 已提交
443

E
emailweixu 已提交
444
  if (averageEvaluator_ &&
445 446
      trainPassContext_.batchId % FLAGS_average_test_period ==
          FLAGS_average_test_period - 1) {
E
emailweixu 已提交
447 448
    averageEvaluator_->finish();
    LOG(INFO) << " Averaged parameter:"
449 450
              << " cost="
              << trainPassContext_.avgTestCost / trainPassContext_.numAvgTests
E
emailweixu 已提交
451 452 453 454
              << " Eval: " << *averageEvaluator_;
    trainPassContext_.numAvgTests = 0;
    trainPassContext_.avgTestCost = 0;
  }
Z
zhangjinchao01 已提交
455

E
emailweixu 已提交
456
  ++trainPassContext_.batchId;
Z
zhangjinchao01 已提交
457

E
emailweixu 已提交
458 459 460 461 462
  if (trainPassContext_.batchId % FLAGS_log_period == 0) {
    FOR_TIMING(globalStat.setThreadInfo(true));
    FOR_TIMING(globalStat.printAllStatus());
    FOR_TIMING(globalStat.reset());
  }
Z
zhangjinchao01 已提交
463

W
wangyanfei01 已提交
464 465 466
  if (testDataProvider_ && FLAGS_test_period > 0 &&
      trainPassContext_.batchId % FLAGS_test_period == 0) {
    tester_->testOnePeriod();
E
emailweixu 已提交
467
  }
Z
zhangjinchao01 已提交
468

E
emailweixu 已提交
469
  if (FLAGS_saving_period_by_batches > 0 &&
470 471
      trainPassContext_.batchId >
          FLAGS_saving_period_by_batches * trainPassContext_.passInnerId &&
E
emailweixu 已提交
472 473 474
      0 == FLAGS_trainer_id) {
    trainerInternal_.getParameterUpdater()->catchUpWith();
    if (testDataProvider_) {
W
wangyanfei01 已提交
475
      tester_->testOnePeriod();
Z
zhangjinchao01 已提交
476
    }
477 478
    paramUtil_->saveParametersOnePass(trainPassContext_.passId,
                                      trainPassContext_.passInnerId);
E
emailweixu 已提交
479
    ++trainPassContext_.passInnerId;
Z
zhangjinchao01 已提交
480
  }
E
emailweixu 已提交
481
}
Z
zhangjinchao01 已提交
482

E
emailweixu 已提交
483 484
void Trainer::finishTrainPass() {
  if (trainPassContext_.batchId == 0) {
Z
zhangjinchao01 已提交
485 486 487 488
    // This means no more data from DataProvider
    return;
  }

489 490
  trainerInternal_.finishTrainPass(trainPassContext_.passId,
                                   trainPassContext_.batchId);
Z
zhangjinchao01 已提交
491 492 493 494 495 496 497 498 499

  FOR_TIMING(globalStat.setThreadInfo(true));
  FOR_TIMING(globalStat.printAllStatus());
  FOR_TIMING(globalStat.reset());

  if (testDataProvider_) {
    tester_->testOnePeriod();
  }

500 501
  if (trainPassContext_.passId % FLAGS_saving_period == 0 &&
      FLAGS_trainer_id == 0) {
E
emailweixu 已提交
502
    paramUtil_->saveParametersOnePass(trainPassContext_.passId);
Z
zhangjinchao01 已提交
503
  }
E
emailweixu 已提交
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523
  ++trainPassContext_.passId;
}

void Trainer::trainOnePass() {
  startTrainPass();
  size_t batchSize = config_->getOptConfig().batch_size();
  while (true) {
    DataBatch dataBatch;

    int num = 0;
    {
      REGISTER_TIMER("getTrainBatch");
      num = dataProvider_->getNextBatch(batchSize, &dataBatch);
    }
    if (num == 0) break;
    CHECK_EQ(num, dataBatch.getSize());
    trainOneDataBatch(dataBatch);
  }

  finishTrainPass();
Z
zhangjinchao01 已提交
524 525 526 527 528 529 530 531 532
}

void Trainer::trainOnePassBatch(int passId) {
  this->stats_->reset();

  trainerInternal_.getParameterUpdater()->startPass();
  const std::vector<Argument> inArgs;
  {
    REGISTER_TIMER("onePass");
533 534
    trainerInternal_.getGradientMachine()->forwardBackward(
        inArgs, nullptr, PASS_TRAIN, nullptr);
Z
zhangjinchao01 已提交
535 536 537 538 539 540 541 542 543
  }

  real cost = .0;
  int64_t num = 0;
  trainerInternal_.getGradientMachine()->getStats(cost, num);
  *stats_ += {num, cost};

  trainerInternal_.getGradientMachine()->onPassEnd();

544
  bool accepted = trainerInternal_.getParameterUpdater()->finishPass(cost);
Z
zhangjinchao01 已提交
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564

  globalStat.setThreadInfo(true);
  globalStat.printAllStatus();
  globalStat.reset();

  LOG(INFO) << " Pass=" << passId
            << " AcceptedPass=" << (accepted ? acceptedPassId_ : -1)
            << stats_->getStats(false /*withCurrentCost*/);

  if (accepted) {
    if (acceptedPassId_ % FLAGS_saving_period == 0 && FLAGS_trainer_id == 0) {
      paramUtil_->saveParameters(acceptedPassId_);
    }
    acceptedPassId_++;
    if (FLAGS_save_only_one && acceptedPassId_ >= FLAGS_saving_period) {
      paramUtil_->deleteParameters(acceptedPassId_ - FLAGS_saving_period);
    }
  }
}

565 566
real Trainer::calcGradient(const DataBatch& dataBatch,
                           const Vector& value,
Z
zhangjinchao01 已提交
567 568 569
                           Vector& gradient) {
  CHECK_EQ(value.getSize(), gradient.getSize());
  std::vector<ParameterPtr>& parameters =
570
      trainerInternal_.getGradientMachine()->getParameters();
Z
zhangjinchao01 已提交
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590

  clearGradient();

  size_t offset = 0;
  size_t valueSize = value.getSize();

  for (auto& para : parameters) {
    CHECK_LE(offset + para->getSize(), valueSize);
    VectorPtr val =
        Vector::create(para->getSize(), value.getMemoryHandle(), offset);
    para->getBuf(PARAMETER_VALUE)->copyFrom(*val);
    para->setValueUpdated();
    offset += para->getSize();
  }

  CHECK_EQ(offset, valueSize);

  std::vector<Argument> inArgs = dataBatch.getStreams();
  std::vector<Argument> outArgs;

591 592
  trainerInternal_.getGradientMachine()->forwardBackward(
      inArgs, &outArgs, PASS_TRAIN);
Z
zhangjinchao01 已提交
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
  real cost = Argument::sumCosts(outArgs);

  offset = 0;
  for (auto& para : parameters) {
    VectorPtr grad =
        Vector::create(para->getSize(), gradient.getMemoryHandle(), offset);
    if (para->getBuf(PARAMETER_GRADIENT)) {
      grad->copyFrom(*para->getBuf(PARAMETER_GRADIENT));
    }
    offset += para->getSize();
  }

  return cost;
}

void Trainer::clearGradient() {
  std::vector<ParameterPtr>& parameters =
      trainerInternal_.getGradientMachine()->getNonStaticParameters();
  for (auto& parameter : parameters) {
    parameter->clearGradient();
  }
}

int Trainer::getBatchSize() { return config_->getOptConfig().batch_size(); }

E
emailweixu 已提交
618
void Trainer::createTester() {
619 620
  tester_.reset(new paddle::Tester(config_,
                                   createTesterConfig(),
E
emailweixu 已提交
621 622 623 624 625
                                   trainerInternal_.getGradientMachine(),
                                   trainerInternal_.getParameterUpdater(),
                                   testDataProvider_));
}

626
void Trainer::test() { tester_->test(); }
Z
zhangjinchao01 已提交
627 628 629

std::unique_ptr<TesterConfig> Trainer::createTesterConfig() {
  TesterConfig* conf = new TesterConfig;
W
wangyanfei01 已提交
630
  if (FLAGS_test_period) {
Y
Yu Yang 已提交
631 632 633 634
    LOG(WARNING) << "The meaning of --test_period is changed: "
                 << "if equal 0, do test on all test data at the end of "
                 << "each pass. While if equal non-zero, do test on all test "
                 << "data every test_period batches ";
W
wangyanfei01 已提交
635 636
  }
  if (FLAGS_test_all_data_in_one_period) {
Y
Yu Yang 已提交
637 638
    LOG(WARNING) << "--test_all_data_in_one_period was deprecated, since "
                 << "we will always do test on all test set ";
W
wangyanfei01 已提交
639
  }
W
wangyanfei01 已提交
640
  conf->testPeriod = FLAGS_test_period;
Z
zhangjinchao01 已提交
641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662
  conf->prevBatchState = FLAGS_prev_batch_state;
  conf->logPeriod = FLAGS_log_period;
  conf->loadsaveParametersInPserver = FLAGS_loadsave_parameters_in_pserver;
  conf->featFile = FLAGS_feat_file;
  conf->predictOutputDir = FLAGS_predict_output_dir;
  conf->trainerId = FLAGS_trainer_id;
  conf->distributeTest = FLAGS_distribute_test;
  conf->config = FLAGS_config;
  conf->modelList = FLAGS_model_list;
  conf->testPass = FLAGS_test_pass;
  conf->numPasses = FLAGS_num_passes;
  conf->savingPeriod = FLAGS_saving_period;
  conf->testWait = FLAGS_test_wait;
  conf->initModelPath = FLAGS_init_model_path;
  conf->saveOnlyOne = FLAGS_save_only_one;
  conf->testing = testing_;
  conf->mode = mode_;
  conf->trainState = &trainState_;
  conf->testState = &testState_;
  return std::unique_ptr<TesterConfig>(conf);
}

663
ParameterUtil* Trainer::getParameterUtilPtr() { return paramUtil_.get(); }
Z
zhangjinchao01 已提交
664
}  // namespace paddle