tester_helper.h 15.2 KB
Newer Older
L
luotao1 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

#include <gtest/gtest.h>
L
luotao1 已提交
18
#include <algorithm>
T
Tao Luo 已提交
19
#include <string>
L
luotao1 已提交
20 21
#include <thread>  // NOLINT
#include <vector>
22

L
luotao1 已提交
23
#include "paddle/fluid/framework/ir/fuse_pass_base.h"
24
#include "paddle/fluid/framework/scope.h"
L
luotao1 已提交
25 26 27 28
#include "paddle/fluid/inference/analysis/analyzer.h"
#include "paddle/fluid/inference/analysis/ut_helper.h"
#include "paddle/fluid/inference/api/analysis_predictor.h"
#include "paddle/fluid/inference/api/paddle_inference_pass.h"
29 30 31

#include "paddle/fluid/inference/api/helper.h"
#include "paddle/fluid/inference/tests/api/config_printer.h"
T
Tao Luo 已提交
32
#include "paddle/fluid/inference/tests/test_helper.h"
N
nhzlx 已提交
33
#include "paddle/fluid/inference/utils/benchmark.h"
L
luotao1 已提交
34 35
#include "paddle/fluid/platform/profiler.h"

N
nhzlx 已提交
36
DEFINE_string(model_name, "", "model name");
L
luotao1 已提交
37 38
DEFINE_string(infer_model, "", "model path");
DEFINE_string(infer_data, "", "data file");
T
Tao Luo 已提交
39
DEFINE_string(refer_result, "", "reference result for comparison");
L
luotao1 已提交
40 41 42 43
DEFINE_int32(batch_size, 1, "batch size.");
DEFINE_int32(repeat, 1, "Running the inference program repeat times.");
DEFINE_bool(test_all_data, false, "Test the all dataset in data file.");
DEFINE_int32(num_threads, 1, "Running the inference program in multi-threads.");
T
Tao Luo 已提交
44 45
DEFINE_bool(use_analysis, true,
            "Running the inference program in analysis mode.");
N
nhzlx 已提交
46 47
DEFINE_bool(record_benchmark, false,
            "Record benchmark after profiling the model");
L
luotao1 已提交
48
DEFINE_double(accuracy, 1e-3, "Result Accuracy.");
L
luotao1 已提交
49

50
DECLARE_bool(profile);
L
luotao1 已提交
51
DECLARE_int32(paddle_num_threads);
52

L
luotao1 已提交
53 54 55
namespace paddle {
namespace inference {

56 57 58 59 60
void PrintConfig(const PaddlePredictor::Config *config, bool use_analysis) {
  if (use_analysis) {
    LOG(INFO) << *reinterpret_cast<const contrib::AnalysisConfig *>(config);
    return;
  }
61
  LOG(INFO) << *reinterpret_cast<const NativeConfig *>(config);
62
}
Y
Yan Chunwei 已提交
63

L
luotao1 已提交
64
void CompareResult(const std::vector<PaddleTensor> &outputs,
T
tensor-tang 已提交
65
                   const std::vector<PaddleTensor> &ref_outputs) {
T
Tao Luo 已提交
66
  EXPECT_GT(outputs.size(), 0UL);
T
tensor-tang 已提交
67
  EXPECT_EQ(outputs.size(), ref_outputs.size());
L
luotao1 已提交
68 69
  for (size_t i = 0; i < outputs.size(); i++) {
    auto &out = outputs[i];
T
tensor-tang 已提交
70
    auto &ref_out = ref_outputs[i];
71 72
    size_t size = VecReduceToInt(out.shape);
    size_t ref_size = VecReduceToInt(ref_out.shape);
73
    EXPECT_GT(size, 0UL);
T
tensor-tang 已提交
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
    EXPECT_EQ(size, ref_size);
    EXPECT_EQ(out.dtype, ref_out.dtype);
    switch (out.dtype) {
      case PaddleDType::INT64: {
        int64_t *pdata = static_cast<int64_t *>(out.data.data());
        int64_t *pdata_ref = static_cast<int64_t *>(ref_out.data.data());
        for (size_t j = 0; j < size; ++j) {
          EXPECT_EQ(pdata_ref[j], pdata[j]);
        }
        break;
      }
      case PaddleDType::FLOAT32: {
        float *pdata = static_cast<float *>(out.data.data());
        float *pdata_ref = static_cast<float *>(ref_out.data.data());
        for (size_t j = 0; j < size; ++j) {
L
luotao1 已提交
89
          EXPECT_NEAR(pdata_ref[j], pdata[j], FLAGS_accuracy);
T
tensor-tang 已提交
90 91 92
        }
        break;
      }
L
luotao1 已提交
93 94 95 96
    }
  }
}

97
std::unique_ptr<PaddlePredictor> CreateTestPredictor(
98
    const PaddlePredictor::Config *config, bool use_analysis = true) {
T
Tao Luo 已提交
99
  if (use_analysis) {
100 101
    return CreatePaddlePredictor<contrib::AnalysisConfig>(
        *(reinterpret_cast<const contrib::AnalysisConfig *>(config)));
T
Tao Luo 已提交
102
  }
103 104
  return CreatePaddlePredictor<NativeConfig>(
      *(reinterpret_cast<const NativeConfig *>(config)));
T
Tao Luo 已提交
105 106
}

107
size_t GetSize(const PaddleTensor &out) { return VecReduceToInt(out.shape); }
T
Tao Luo 已提交
108

109
std::unordered_map<std::string, int> GetFuseStatis(PaddlePredictor *predictor,
T
Tao Luo 已提交
110
                                                   int *num_ops) {
111
  std::unordered_map<std::string, int> res;
112
  auto *analysis_predictor = static_cast<AnalysisPredictor *>(predictor);
113 114 115 116 117 118
  auto *fusion_status =
      analysis_predictor->analysis_argument().fusion_statis_ptr();
  if (!fusion_status) {
    return res;
  }
  for (auto &item : *fusion_status) {
T
Tao Luo 已提交
119 120 121 122
    LOG(INFO) << "fused " << item.first << " " << item.second;
  }
  int num = 0;
  for (auto &node :
123 124
       analysis_predictor->analysis_argument().main_graph().Nodes()) {
    if (node->IsOp()) {
T
Tao Luo 已提交
125 126 127 128
      ++num;
    }
  }
  *num_ops = num;
129
  return *fusion_status;
T
Tao Luo 已提交
130 131
}

T
Tao Luo 已提交
132
void SetFakeImageInput(std::vector<std::vector<PaddleTensor>> *inputs,
133 134
                       const std::string &dirname, bool is_combined = true,
                       std::string model_filename = "model",
T
tensor-tang 已提交
135 136
                       std::string params_filename = "params",
                       const std::vector<std::string> *feed_names = nullptr) {
T
Tao Luo 已提交
137 138
  // Set fake_image_data
  PADDLE_ENFORCE_EQ(FLAGS_test_all_data, 0, "Only have single batch of data.");
139 140 141 142 143 144 145 146 147 148 149
  std::vector<std::vector<int64_t>> feed_target_shapes = GetFeedTargetShapes(
      dirname, is_combined, model_filename, params_filename);
  std::ostringstream os;
  for (size_t i = 0; i < feed_target_shapes.size(); ++i) {
    os << "feed target " << i << ": {" << feed_target_shapes[i][0];
    for (size_t j = 1; j < feed_target_shapes[i].size(); ++j) {
      os << ", " << feed_target_shapes[i][j];
    }
    os << "}\n";
  }
  LOG(INFO) << os.str();
T
tensor-tang 已提交
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
  if (feed_names) {
    PADDLE_ENFORCE_EQ(feed_names->size(), feed_target_shapes.size());
  }
  std::vector<PaddleTensor> input_slots(feed_target_shapes.size());
  for (size_t i = 0; i < feed_target_shapes.size(); ++i) {
    const auto &feed_shape = feed_target_shapes[i];
    auto &input = input_slots[i];
    std::vector<int> shape({FLAGS_batch_size});
    for (size_t s = 1; s < feed_shape.size(); ++s) {
      shape.push_back(static_cast<int>(feed_shape[s]));
    }
    if (feed_names) {
      input.name = (*feed_names)[i];
    }
    input.shape = shape;
    input.dtype = PaddleDType::FLOAT32;
    size_t len = std::accumulate(shape.begin(), shape.end(), 1,
                                 [](int a, int b) { return a * b; });
    input.data.Resize(len * sizeof(float));
    input.lod.assign({{0, static_cast<size_t>(FLAGS_batch_size)}});
    float *input_data = static_cast<float *>(input.data.data());
    // fill input data, for profile easily, do not use random data here.
    for (size_t j = 0; j < len; ++j) {
      *(input_data + j) = static_cast<float>(j) / len;
    }
T
Tao Luo 已提交
175 176 177 178
  }
  (*inputs).emplace_back(input_slots);
}

179 180 181 182 183 184 185 186 187 188 189 190
void GetInputPerBatch(const std::vector<std::vector<int64_t>> &in,
                      std::vector<std::vector<int64_t>> *out,
                      std::vector<size_t> *lod, size_t batch_iter,
                      size_t batch_end) {
  lod->clear();
  lod->push_back(0);
  for (auto it = in.begin() + batch_iter; it < in.begin() + batch_end; it++) {
    out->push_back(*it);
    lod->push_back(lod->back() + (*it).size());  // calculate lod
  }
}

L
luotao1 已提交
191
void TestOneThreadPrediction(
192
    const PaddlePredictor::Config *config,
193
    const std::vector<std::vector<PaddleTensor>> &inputs,
T
Tao Luo 已提交
194
    std::vector<PaddleTensor> *outputs, bool use_analysis = true) {
L
luotao1 已提交
195 196
  int batch_size = FLAGS_batch_size;
  int num_times = FLAGS_repeat;
197
  auto predictor = CreateTestPredictor(config, use_analysis);
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218

  // warmup run
  LOG(INFO) << "Warm up run...";
  {
    Timer warmup_timer;
    warmup_timer.tic();
    predictor->Run(inputs[0], outputs, batch_size);
    PrintTime(batch_size, 1, 1, 0, warmup_timer.toc(), 1);
    if (FLAGS_profile) {
      paddle::platform::ResetProfiler();
    }
  }

  LOG(INFO) << "Run " << num_times << " times...";
  {
    Timer run_timer;
    run_timer.tic();
    for (int i = 0; i < num_times; i++) {
      for (size_t j = 0; j < inputs.size(); j++) {
        predictor->Run(inputs[j], outputs, batch_size);
      }
L
luotao1 已提交
219
    }
N
nhzlx 已提交
220 221 222 223 224 225 226 227 228 229

    double latency = run_timer.toc() / num_times;
    PrintTime(batch_size, num_times, 1, 0, latency, inputs.size());
    if (FLAGS_record_benchmark) {
      Benchmark benchmark;
      benchmark.SetName(FLAGS_model_name);
      benchmark.SetBatchSize(batch_size);
      benchmark.SetLatency(latency);
      benchmark.PersistToFile("benchmark_record.txt");
    }
L
luotao1 已提交
230 231 232 233
  }
}

void TestMultiThreadPrediction(
234
    const PaddlePredictor::Config *config,
235
    const std::vector<std::vector<PaddleTensor>> &inputs,
T
Tao Luo 已提交
236 237
    std::vector<PaddleTensor> *outputs, int num_threads,
    bool use_analysis = true) {
L
luotao1 已提交
238 239 240
  int batch_size = FLAGS_batch_size;
  int num_times = FLAGS_repeat;
  std::vector<std::thread> threads;
241
  auto main_predictor = CreateTestPredictor(config, use_analysis);
242 243

  size_t total_time{0};
L
luotao1 已提交
244 245 246 247 248
  for (int tid = 0; tid < num_threads; ++tid) {
    threads.emplace_back([&, tid]() {
      // Each thread should have local inputs and outputs.
      // The inputs of each thread are all the same.
      std::vector<PaddleTensor> outputs_tid;
249 250 251
      // To ensure the thread binding correctly,
      // please clone inside the threadpool.
      auto predictor = main_predictor->Clone();
L
luotao1 已提交
252 253 254
#ifdef PADDLE_WITH_MKLDNN
      if (use_analysis) {
        static_cast<AnalysisPredictor *>(predictor.get())
L
luotao1 已提交
255
            ->SetMkldnnThreadID(static_cast<int>(tid) + 1);
L
luotao1 已提交
256 257
      }
#endif
T
Tao Luo 已提交
258 259 260 261 262 263 264 265 266 267

      // warmup run
      LOG(INFO) << "Running thread " << tid << ", warm up run...";
      {
        Timer warmup_timer;
        warmup_timer.tic();
        predictor->Run(inputs[0], outputs, batch_size);
        PrintTime(batch_size, 1, num_threads, tid, warmup_timer.toc(), 1);
        if (FLAGS_profile) {
          paddle::platform::ResetProfiler();
L
luotao1 已提交
268 269
        }
      }
270

T
Tao Luo 已提交
271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
      LOG(INFO) << "Thread " << tid << " run " << num_times << " times...";
      {
        Timer timer;
        timer.tic();
        for (int i = 0; i < num_times; i++) {
          for (const auto &input : inputs) {
            ASSERT_TRUE(predictor->Run(input, &outputs_tid));
          }
        }

        auto time = timer.toc();
        total_time += time;
        PrintTime(batch_size, num_times, num_threads, tid, time / num_times,
                  inputs.size());
      }
L
luotao1 已提交
286 287 288 289 290 291 292
    });
  }
  for (int i = 0; i < num_threads; ++i) {
    threads[i].join();
  }
}

293
void TestPrediction(const PaddlePredictor::Config *config,
294
                    const std::vector<std::vector<PaddleTensor>> &inputs,
T
Tao Luo 已提交
295 296
                    std::vector<PaddleTensor> *outputs, int num_threads,
                    bool use_analysis = FLAGS_use_analysis) {
297
  PrintConfig(config, use_analysis);
L
luotao1 已提交
298
  if (num_threads == 1) {
T
Tao Luo 已提交
299
    TestOneThreadPrediction(config, inputs, outputs, use_analysis);
L
luotao1 已提交
300
  } else {
T
Tao Luo 已提交
301 302
    TestMultiThreadPrediction(config, inputs, outputs, num_threads,
                              use_analysis);
L
luotao1 已提交
303 304 305
  }
}

L
luotao1 已提交
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
void CompareDeterministic(
    const PaddlePredictor::Config *config,
    const std::vector<std::vector<PaddleTensor>> &inputs) {
  int batch_size = FLAGS_batch_size;
  int num_times = FLAGS_repeat;
  auto predictor = CreateTestPredictor(config, FLAGS_use_analysis);

  // warmup run
  std::vector<PaddleTensor> warmup_outputs, outputs;
  predictor->Run(inputs[0], &warmup_outputs, batch_size);

  // run num_times to Compare Deterministic Result.
  for (int i = 0; i < num_times; i++) {
    for (size_t j = 0; j < inputs.size(); j++) {
      predictor->Run(inputs[j], &outputs, batch_size);
      CompareResult(outputs, warmup_outputs);
    }
  }
}

T
Tao Luo 已提交
326
void CompareNativeAndAnalysis(
327
    const PaddlePredictor::Config *config,
328
    const std::vector<std::vector<PaddleTensor>> &inputs) {
329
  PrintConfig(config, true);
T
Tao Luo 已提交
330 331 332 333 334 335
  std::vector<PaddleTensor> native_outputs, analysis_outputs;
  TestOneThreadPrediction(config, inputs, &native_outputs, false);
  TestOneThreadPrediction(config, inputs, &analysis_outputs, true);
  CompareResult(analysis_outputs, native_outputs);
}

L
luotao1 已提交
336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416
template <typename T>
std::string LoDTensorSummary(const framework::LoDTensor &tensor) {
  std::stringstream ss;
  ss << "\n---- tensor ---" << '\n';
  ss << "lod: [";
  for (const auto &level : tensor.lod()) {
    ss << "[ ";
    for (auto i : level) {
      ss << i << ", ";
    }
    ss << "]";
  }
  ss << "]\n";

  ss << "shape: [";
  int size = 1;
  for (int i = 0; i < tensor.dims().size(); i++) {
    int dim = tensor.dims()[i];
    ss << dim << ", ";
    size *= dim;
  }
  ss << "]\n";

  ss << "data: ";
  for (int i = 0; i < std::min(20, size); i++) {
    ss << tensor.data<T>()[i] << " ";
  }
  ss << "\n";

  return ss.str();
}

static bool CompareLoD(const framework::LoD &a, const framework::LoD &b) {
  if (a.size() != b.size()) {
    LOG(ERROR) << string::Sprintf("lod size not match %d != %d", a.size(),
                                  b.size());
    return false;
  }
  for (size_t i = 0; i < a.size(); i++) {
    auto &al = a[i];
    auto &bl = b[i];
    if (al.size() != bl.size()) {
      LOG(ERROR) << string::Sprintf("level size %d != %d", al.size(),
                                    bl.size());
      return false;
    }
  }
  return true;
}

static bool CompareShape(const std::vector<int64_t> &a,
                         const std::vector<int64_t> &b) {
  if (a.size() != b.size()) {
    LOG(ERROR) << string::Sprintf("shape size not match %d != %d", a.size(),
                                  b.size());
    return false;
  }
  for (size_t i = 0; i < a.size(); i++) {
    if (a[i] != b[i]) {
      LOG(ERROR) << string::Sprintf("shape %d-th element not match %d != %d", i,
                                    a[i], b[i]);
      return false;
    }
  }
  return true;
}

static bool CompareTensorData(const framework::LoDTensor &a,
                              const framework::LoDTensor &b) {
  auto a_shape = framework::vectorize(a.dims());
  auto b_shape = framework::vectorize(b.dims());
  size_t a_size = std::accumulate(a_shape.begin(), a_shape.end(), 1,
                                  [](int a, int b) { return a * b; });
  size_t b_size = std::accumulate(b_shape.begin(), b_shape.end(), 1,
                                  [](int a, int b) { return a * b; });
  if (a_size != b_size) {
    LOG(ERROR) << string::Sprintf("tensor data size not match, %d != %d",
                                  a_size, b_size);
  }

  for (size_t i = 0; i < a_size; i++) {
Y
Yu Yang 已提交
417
    if (a.type() == framework::proto::VarType::FP32) {
L
luotao1 已提交
418 419 420 421 422 423 424 425
      const auto *a_data = a.data<float>();
      const auto *b_data = b.data<float>();
      if (std::abs(a_data[i] - b_data[i]) > 1e-3) {
        LOG(ERROR) << string::Sprintf(
            "tensor data %d-th element not match, %f != %f", i, a_data[i],
            b_data[i]);
        return false;
      }
Y
Yu Yang 已提交
426
    } else if (a.type() == framework::proto::VarType::INT64) {
L
luotao1 已提交
427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
      const auto *a_data = a.data<int64_t>();
      const auto *b_data = b.data<int64_t>();
      if (std::abs(a_data[i] - b_data[i]) > 1e-3) {
        LOG(ERROR) << string::Sprintf(
            "tensor data %d-th element not match, %f != %f", i, a_data[i],
            b_data[i]);
        return false;
      }
    }
  }

  return true;
}

static bool CompareTensor(const framework::LoDTensor &a,
                          const framework::LoDTensor &b) {
  if (!CompareLoD(a.lod(), b.lod())) {
    return false;
  }
  if (!CompareShape(framework::vectorize(a.dims()),
                    framework::vectorize(b.dims()))) {
    return false;
  }

  if (!CompareTensorData(a, b)) {
    return false;
  }

  return true;
}

L
luotao1 已提交
458 459
}  // namespace inference
}  // namespace paddle