legacy_allocator.cc 12.6 KB
Newer Older
Y
Yu Yang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

15
#include <memory>
Y
Yu Yang 已提交
16
#include <string>
L
liuwei1031 已提交
17
#include <utility>
18
#include <vector>
M
minqiyang 已提交
19

20
#ifdef PADDLE_WITH_JEMALLOC
M
minqiyang 已提交
21 22 23
#include <jemalloc/jemalloc.h>
#endif

Y
Yu Yang 已提交
24
#include "glog/logging.h"
C
chengduo 已提交
25
#include "paddle/fluid/memory/allocation/legacy_allocator.h"
Y
Yu Yang 已提交
26 27 28
#include "paddle/fluid/memory/detail/buddy_allocator.h"
#include "paddle/fluid/memory/detail/system_allocator.h"
#include "paddle/fluid/platform/gpu_info.h"
C
chengduo 已提交
29
#include "paddle/fluid/platform/profiler.h"
Y
Yu Yang 已提交
30
#include "paddle/fluid/string/printf.h"
31
#include "paddle/fluid/string/split.h"
Y
Yu Yang 已提交
32 33 34 35 36 37 38 39

DEFINE_bool(init_allocated_mem, false,
            "It is a mistake that the values of the memory allocated by "
            "BuddyAllocator are always zeroed in some op's implementation. "
            "To find this error in time, we use init_allocated_mem to indicate "
            "that initializing the allocated memory with a small value "
            "during unit testing.");
DECLARE_double(fraction_of_gpu_memory_to_use);
40 41
DECLARE_uint64(initial_gpu_memory_in_mb);
DECLARE_uint64(reallocate_gpu_memory_in_mb);
D
dzhwinter 已提交
42
DECLARE_bool(benchmark);
Y
Yu Yang 已提交
43 44 45 46 47 48 49 50

namespace paddle {
namespace memory {
namespace legacy {
template <typename Place>
void *Alloc(const Place &place, size_t size);

template <typename Place>
L
liuwei1031 已提交
51
void Free(const Place &place, void *p, size_t size);
Y
Yu Yang 已提交
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74

template <typename Place>
size_t Used(const Place &place);

struct Usage : public boost::static_visitor<size_t> {
  size_t operator()(const platform::CPUPlace &cpu) const;
  size_t operator()(const platform::CUDAPlace &gpu) const;
  size_t operator()(const platform::CUDAPinnedPlace &cuda_pinned) const;
};

size_t memory_usage(const platform::Place &p);

using BuddyAllocator = detail::BuddyAllocator;

BuddyAllocator *GetCPUBuddyAllocator() {
  // We tried thread_local for inference::RNN1 model, but that not works much
  // for multi-thread test.
  static std::once_flag init_flag;
  static detail::BuddyAllocator *a = nullptr;

  std::call_once(init_flag, []() {
    a = new detail::BuddyAllocator(
        std::unique_ptr<detail::SystemAllocator>(new detail::CPUAllocator),
S
sneaxiy 已提交
75
        platform::CpuMinChunkSize(), platform::CpuMaxChunkSize());
Y
Yu Yang 已提交
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
  });

  return a;
}

// We compared the NaiveAllocator with BuddyAllocator in CPU memory allocation,
// seems they are almost the same overhead.
struct NaiveAllocator {
  void *Alloc(size_t size) { return malloc(size); }

  void Free(void *p) {
    PADDLE_ENFORCE(p);
    free(p);
  }

  static NaiveAllocator *Instance() {
    static NaiveAllocator x;
    return &x;
  }

 private:
  std::mutex lock_;
};

template <>
void *Alloc<platform::CPUPlace>(const platform::CPUPlace &place, size_t size) {
G
gongweibao 已提交
102
  VLOG(10) << "Allocate " << size << " bytes on " << platform::Place(place);
103
#ifdef PADDLE_WITH_JEMALLOC
M
minqiyang 已提交
104 105
  void *p = malloc(size);
#else
Y
Yu Yang 已提交
106
  void *p = GetCPUBuddyAllocator()->Alloc(size);
M
minqiyang 已提交
107
#endif
Y
Yu Yang 已提交
108 109 110
  if (FLAGS_init_allocated_mem) {
    memset(p, 0xEF, size);
  }
M
minqiyang 已提交
111
  VLOG(10) << "  pointer=" << p;
Y
Yu Yang 已提交
112 113 114 115
  return p;
}

template <>
L
liuwei1031 已提交
116 117
void Free<platform::CPUPlace>(const platform::CPUPlace &place, void *p,
                              size_t size) {
G
gongweibao 已提交
118
  VLOG(10) << "Free pointer=" << p << " on " << platform::Place(place);
119
#ifdef PADDLE_WITH_JEMALLOC
M
minqiyang 已提交
120 121
  free(p);
#else
Y
Yu Yang 已提交
122
  GetCPUBuddyAllocator()->Free(p);
M
minqiyang 已提交
123
#endif
Y
Yu Yang 已提交
124 125 126 127
}

template <>
size_t Used<platform::CPUPlace>(const platform::CPUPlace &place) {
128 129
#ifdef PADDLE_WITH_JEMALLOC
  // fake the result of used memory when PADDLE_WITH_JEMALLOC is ON
M
minqiyang 已提交
130 131
  return 0U;
#else
Y
Yu Yang 已提交
132
  return GetCPUBuddyAllocator()->Used();
M
minqiyang 已提交
133
#endif
Y
Yu Yang 已提交
134 135 136
}

#ifdef PADDLE_WITH_CUDA
S
sneaxiy 已提交
137 138 139 140 141 142 143
class GPUBuddyAllocatorList {
 public:
  GPUBuddyAllocatorList()
      : allocators_(platform::GetCUDADeviceCount()),
        flags_(platform::GetCUDADeviceCount()) {
    allocation::GPUMemMonitor.Initialize(allocators_.size());
  }
144

S
sneaxiy 已提交
145 146 147
  BuddyAllocator *Get(size_t dev_id) {
    PADDLE_ENFORCE(dev_id < flags_.size(), "Invalid device id %s", dev_id);
    std::call_once(flags_[dev_id], [this, dev_id] {
148
      platform::SetDeviceId(dev_id);
S
sneaxiy 已提交
149 150 151 152
      allocators_[dev_id] = new BuddyAllocator(
          std::unique_ptr<detail::SystemAllocator>(
              new detail::GPUAllocator(dev_id)),
          platform::GpuMinChunkSize(), platform::GpuMaxChunkSize());
Y
Yu Yang 已提交
153

S
sneaxiy 已提交
154 155 156 157 158 159 160 161 162 163 164 165
      VLOG(10) << "\n\nNOTE:\n"
               << "You can set GFlags environment variable "
               << "'FLAGS_fraction_of_gpu_memory_to_use' "
               << "or 'FLAGS_initial_gpu_memory_in_mb' "
               << "or 'FLAGS_reallocate_gpu_memory_in_mb' "
               << "to change the memory size for GPU usage.\n"
               << "Current 'FLAGS_fraction_of_gpu_memory_to_use' value is "
               << FLAGS_fraction_of_gpu_memory_to_use
               << ". Current 'FLAGS_initial_gpu_memory_in_mb' value is "
               << FLAGS_initial_gpu_memory_in_mb
               << ". Current 'FLAGS_reallocate_gpu_memory_in_mb' value is "
               << FLAGS_reallocate_gpu_memory_in_mb << "\n\n";
S
sneaxiy 已提交
166 167 168 169 170 171 172 173
    });
    return allocators_[dev_id];
  }

 private:
  std::vector<BuddyAllocator *> allocators_;
  std::vector<std::once_flag> flags_;
};
Y
Yu Yang 已提交
174

S
sneaxiy 已提交
175 176
BuddyAllocator *GetGPUBuddyAllocator(int gpu_id) {
  static GPUBuddyAllocatorList allocators;
Y
Yu Yang 已提交
177
  platform::SetDeviceId(gpu_id);
S
sneaxiy 已提交
178
  return allocators.Get(gpu_id);
Y
Yu Yang 已提交
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
}
#endif

template <>
size_t Used<platform::CUDAPlace>(const platform::CUDAPlace &place) {
#ifdef PADDLE_WITH_CUDA
  return GetGPUBuddyAllocator(place.device)->Used();
#else
  PADDLE_THROW("'CUDAPlace' is not supported in CPU only device.");
#endif
}

template <>
void *Alloc<platform::CUDAPlace>(const platform::CUDAPlace &place,
                                 size_t size) {
#ifdef PADDLE_WITH_CUDA
  auto *buddy_allocator = GetGPUBuddyAllocator(place.device);
  auto *ptr = buddy_allocator->Alloc(size);
S
sneaxiy 已提交
197
  if (ptr == nullptr && size > 0) {
Y
Yu Yang 已提交
198 199 200 201
    int cur_dev = platform::GetCurrentDeviceId();
    platform::SetDeviceId(place.device);
    size_t avail, total;
    platform::GpuMemoryUsage(&avail, &total);
D
dzhwinter 已提交
202 203 204 205 206 207 208 209 210
    LOG(FATAL) << "Cannot allocate " << string::HumanReadableSize(size)
               << " in GPU " << place.device << ", available "
               << string::HumanReadableSize(avail) << "total " << total
               << "GpuMinChunkSize "
               << string::HumanReadableSize(buddy_allocator->GetMinChunkSize())
               << "GpuMaxChunkSize "
               << string::HumanReadableSize(buddy_allocator->GetMaxChunkSize())
               << "GPU memory used: "
               << string::HumanReadableSize(Used<platform::CUDAPlace>(place));
Y
Yu Yang 已提交
211
    platform::SetDeviceId(cur_dev);
L
liuwei1031 已提交
212
  } else {
D
dzhwinter 已提交
213
    if (FLAGS_benchmark) {
D
dzhwinter 已提交
214 215
      allocation::GPUMemMonitor.Add(place.device, size);
    }
L
liuwei1031 已提交
216 217 218
    if (FLAGS_init_allocated_mem) {
      cudaMemset(ptr, 0xEF, size);
    }
Y
Yu Yang 已提交
219 220 221 222 223 224 225 226
  }
  return ptr;
#else
  PADDLE_THROW("'CUDAPlace' is not supported in CPU only device.");
#endif
}

template <>
L
liuwei1031 已提交
227 228
void Free<platform::CUDAPlace>(const platform::CUDAPlace &place, void *p,
                               size_t size) {
Y
Yu Yang 已提交
229 230
#ifdef PADDLE_WITH_CUDA
  GetGPUBuddyAllocator(place.device)->Free(p);
D
dzhwinter 已提交
231
  if (FLAGS_benchmark) {
D
dzhwinter 已提交
232 233
    allocation::GPUMemMonitor.Minus(place.device, size);
  }
Y
Yu Yang 已提交
234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
#else
  PADDLE_THROW("'CUDAPlace' is not supported in CPU only device.");
#endif
}

#ifdef PADDLE_WITH_CUDA
BuddyAllocator *GetCUDAPinnedBuddyAllocator() {
  static std::once_flag init_flag;
  static BuddyAllocator *ba = nullptr;

  std::call_once(init_flag, []() {
    ba = new BuddyAllocator(std::unique_ptr<detail::SystemAllocator>(
                                new detail::CUDAPinnedAllocator),
                            platform::CUDAPinnedMinChunkSize(),
                            platform::CUDAPinnedMaxChunkSize());
  });

  return ba;
}
#endif

template <>
size_t Used<platform::CUDAPinnedPlace>(const platform::CUDAPinnedPlace &place) {
#ifdef PADDLE_WITH_CUDA
  return GetCUDAPinnedBuddyAllocator()->Used();
#else
  PADDLE_THROW("'CUDAPinnedPlace' is not supported in CPU only device.");
#endif
}

template <>
void *Alloc<platform::CUDAPinnedPlace>(const platform::CUDAPinnedPlace &place,
                                       size_t size) {
#ifdef PADDLE_WITH_CUDA
  auto *buddy_allocator = GetCUDAPinnedBuddyAllocator();
  void *ptr = buddy_allocator->Alloc(size);

  if (ptr == nullptr) {
D
Dun Liang 已提交
272
    LOG(WARNING) << "cudaHostAlloc Cannot allocate " << size
Y
Yu Yang 已提交
273 274 275 276 277 278 279 280 281 282 283 284 285
                 << " bytes in CUDAPinnedPlace";
  }
  if (FLAGS_init_allocated_mem) {
    memset(ptr, 0xEF, size);
  }
  return ptr;
#else
  PADDLE_THROW("'CUDAPinnedPlace' is not supported in CPU only device.");
#endif
}

template <>
void Free<platform::CUDAPinnedPlace>(const platform::CUDAPinnedPlace &place,
L
liuwei1031 已提交
286
                                     void *p, size_t size) {
Y
Yu Yang 已提交
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
#ifdef PADDLE_WITH_CUDA
  GetCUDAPinnedBuddyAllocator()->Free(p);
#else
  PADDLE_THROW("'CUDAPinnedPlace' is not supported in CPU only device.");
#endif
}

struct AllocVisitor : public boost::static_visitor<void *> {
  inline explicit AllocVisitor(size_t size) : size_(size) {}

  template <typename Place>
  inline void *operator()(const Place &place) const {
    return Alloc<Place>(place, size_);
  }

 private:
  size_t size_;
};

struct FreeVisitor : public boost::static_visitor<void> {
L
liuwei1031 已提交
307 308
  inline explicit FreeVisitor(void *ptr, size_t size)
      : ptr_(ptr), size_(size) {}
Y
Yu Yang 已提交
309 310 311

  template <typename Place>
  inline void operator()(const Place &place) const {
L
liuwei1031 已提交
312
    Free<Place>(place, ptr_, size_);
Y
Yu Yang 已提交
313 314 315 316
  }

 private:
  void *ptr_;
L
liuwei1031 已提交
317
  size_t size_;
Y
Yu Yang 已提交
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
};

size_t Usage::operator()(const platform::CPUPlace &cpu) const {
  return Used(cpu);
}

size_t Usage::operator()(const platform::CUDAPlace &gpu) const {
#ifdef PADDLE_WITH_CUDA
  return Used(gpu);
#else
  PADDLE_THROW("'CUDAPlace' is not supported in CPU only device.");
#endif
}

size_t Usage::operator()(const platform::CUDAPinnedPlace &cuda_pinned) const {
#ifdef PADDLE_WITH_CUDA
  return Used(cuda_pinned);
#else
  PADDLE_THROW("'CUDAPinnedPlace' is not supported in CPU only device.");
#endif
}
}  // namespace legacy

namespace allocation {
342 343
LegacyMemMonitor GPUMemMonitor;

Y
Yu Yang 已提交
344 345
Allocation *LegacyAllocator::AllocateImpl(size_t size, Allocator::Attr attr) {
  void *ptr = boost::apply_visitor(legacy::AllocVisitor(size), place_);
C
chengduo 已提交
346 347 348 349
  auto *tmp_alloc = new Allocation(ptr, size, place_);
  platform::MemEvenRecorder::Instance().PushMemRecord(
      static_cast<void *>(tmp_alloc), place_, size);
  return tmp_alloc;
Y
Yu Yang 已提交
350 351
}

S
sneaxiy 已提交
352
void LegacyAllocator::FreeImpl(Allocation *allocation) {
L
liuwei1031 已提交
353 354 355
  boost::apply_visitor(
      legacy::FreeVisitor(allocation->ptr(), allocation->size()),
      allocation->place());
C
chengduo 已提交
356 357
  platform::MemEvenRecorder::Instance().PopMemRecord(
      static_cast<void *>(allocation), place_);
Y
Yu Yang 已提交
358 359
  delete allocation;
}
360 361 362 363 364 365 366 367 368 369 370 371 372 373

bool MemInfo::Add(const size_t &size) {
  std::lock_guard<std::mutex> lock(mutex_);
  usage_ += size;
  bool peak_point = usage_ > peak_usage_;
  if (peak_point) peak_usage_ = usage_;
  return peak_point;
}

void MemInfo::Minus(const size_t &size) {
  std::lock_guard<std::mutex> lock(mutex_);
  usage_ -= size;
}

374
uint64_t MemInfo::GetPeakUsage() const { return peak_usage_; }
375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397

LegacyMemMonitor::~LegacyMemMonitor() {
  for (auto &item : gpu_mem_info_) delete item.second;
}

void LegacyMemMonitor::Initialize(const int &device_num) {
  for (auto i = 0; i < device_num; ++i) {
    gpu_mem_info_[i] = new MemInfo();
  }
}

void LegacyMemMonitor::Add(const int &device, const size_t &size) {
  if (gpu_mem_info_[device]->Add(size)) {
    VLOG(3) << "#LegacyMemMonitor# device: " << device
            << " peak memory usage : "
            << (gpu_mem_info_[device]->GetPeakUsage() >> 20) << " MiB";
  }
}

void LegacyMemMonitor::Minus(const int &device, const size_t &size) {
  gpu_mem_info_[device]->Minus(size);
}

398
uint64_t LegacyMemMonitor::GetMemUsage(const int &device) const {
399 400
  return gpu_mem_info_.find(device) == gpu_mem_info_.end()
             ? 0
L
liuwei1031 已提交
401
             : gpu_mem_info_.at(device)->GetPeakUsage();
402 403 404 405 406 407 408 409 410 411 412 413 414 415 416
}

void LegacyMemMonitor::PrintMemUsage() {
  std::vector<int> devices;
  for (const auto &item : gpu_mem_info_) {
    devices.emplace_back(item.first);
  }
  std::sort(devices.begin(), devices.end());
  for (const auto &device : devices) {
    std::cout << "Device : " << device << " Peak Memory Usage : "
              << (gpu_mem_info_[device]->GetPeakUsage() >> 20) << " MiB"
              << std::endl;
  }
}

Y
Yu Yang 已提交
417 418 419
}  // namespace allocation
}  // namespace memory
}  // namespace paddle