env_win.cc 30.6 KB
Newer Older
1
//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
S
Siying Dong 已提交
2 3 4
//  This source code is licensed under both the GPLv2 (found in the
//  COPYING file in the root directory) and Apache 2.0 License
//  (found in the LICENSE.Apache file in the root directory).
D
Dmitri Smirnov 已提交
5 6 7 8 9
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.

A
Aaron Gao 已提交
10
#include "port/win/env_win.h"
D
Dmitri Smirnov 已提交
11
#include "port/win/win_thread.h"
D
Dmitri Smirnov 已提交
12 13
#include <algorithm>
#include <ctime>
A
Aaron Gao 已提交
14
#include <thread>
D
Dmitri Smirnov 已提交
15 16

#include <errno.h>
17 18 19
#include <process.h> // _getpid
#include <io.h> // _access
#include <direct.h> // _rmdir, _mkdir, _getcwd
D
Dmitri Smirnov 已提交
20 21 22 23 24 25 26 27 28
#include <sys/types.h>
#include <sys/stat.h>

#include "rocksdb/env.h"
#include "rocksdb/slice.h"

#include "port/port.h"
#include "port/dirent.h"
#include "port/win/win_logger.h"
29
#include "port/win/io_win.h"
D
Dmitri Smirnov 已提交
30

31
#include "monitoring/iostats_context_imp.h"
D
Dmitri Smirnov 已提交
32

33 34
#include "monitoring/thread_status_updater.h"
#include "monitoring/thread_status_util.h"
D
Dmitri Smirnov 已提交
35

36 37
#include <rpc.h>  // for uuid generation
#include <windows.h>
D
Dmitri Smirnov 已提交
38

S
sdong 已提交
39
namespace rocksdb {
D
Dmitri Smirnov 已提交
40 41 42 43 44

ThreadStatusUpdater* CreateThreadStatusUpdater() {
  return new ThreadStatusUpdater();
}

D
Dmitri Smirnov 已提交
45 46
namespace {

D
Dmitri Smirnov 已提交
47 48 49 50
// RAII helpers for HANDLEs
const auto CloseHandleFunc = [](HANDLE h) { ::CloseHandle(h); };
typedef std::unique_ptr<void, decltype(CloseHandleFunc)> UniqueCloseHandlePtr;

51 52 53 54
void WinthreadCall(const char* label, std::error_code result) {
  if (0 != result.value()) {
    fprintf(stderr, "pthread %s: %s\n", label, strerror(result.value()));
    abort();
D
Dmitri Smirnov 已提交
55
  }
56
}
D
Dmitri Smirnov 已提交
57 58 59

}

60
namespace port {
D
Dmitri Smirnov 已提交
61

62 63 64 65 66 67
WinEnvIO::WinEnvIO(Env* hosted_env)
  :   hosted_env_(hosted_env),
      page_size_(4 * 1012),
      allocation_granularity_(page_size_),
      perf_counter_frequency_(0),
      GetSystemTimePreciseAsFileTime_(NULL) {
D
Dmitri Smirnov 已提交
68

69 70
  SYSTEM_INFO sinfo;
  GetSystemInfo(&sinfo);
D
Dmitri Smirnov 已提交
71

72 73
  page_size_ = sinfo.dwPageSize;
  allocation_granularity_ = sinfo.dwAllocationGranularity;
D
Dmitri Smirnov 已提交
74

75 76
  {
    LARGE_INTEGER qpf;
77 78 79
    // No init as the compiler complains about unused var
    BOOL ret;
    ret = QueryPerformanceFrequency(&qpf);
80 81
    assert(ret == TRUE);
    perf_counter_frequency_ = qpf.QuadPart;
D
Dmitri Smirnov 已提交
82 83
  }

84 85 86 87
  HMODULE module = GetModuleHandle("kernel32.dll");
  if (module != NULL) {
    GetSystemTimePreciseAsFileTime_ = (FnGetSystemTimePreciseAsFileTime)GetProcAddress(
      module, "GetSystemTimePreciseAsFileTime");
D
Dmitri Smirnov 已提交
88 89 90
  }
}

91
WinEnvIO::~WinEnvIO() {
D
Dmitri Smirnov 已提交
92 93
}

94 95
Status WinEnvIO::DeleteFile(const std::string& fname) {
  Status result;
D
Dmitri Smirnov 已提交
96

97 98
  if (_unlink(fname.c_str())) {
    result = IOError("Failed to delete: " + fname, errno);
D
Dmitri Smirnov 已提交
99 100
  }

101
  return result;
D
Dmitri Smirnov 已提交
102 103
}

104 105 106 107
Status WinEnvIO::GetCurrentTime(int64_t* unix_time) {
  time_t time = std::time(nullptr);
  if (time == (time_t)(-1)) {
    return Status::NotSupported("Failed to get time");
D
Dmitri Smirnov 已提交
108 109
  }

110 111
  *unix_time = time;
  return Status::OK();
D
Dmitri Smirnov 已提交
112 113
}

114 115 116 117
Status WinEnvIO::NewSequentialFile(const std::string& fname,
  std::unique_ptr<SequentialFile>* result,
  const EnvOptions& options) {
  Status s;
D
Dmitri Smirnov 已提交
118

119
  result->reset();
D
Dmitri Smirnov 已提交
120

121 122 123 124
  // Corruption test needs to rename and delete files of these kind
  // while they are still open with another handle. For that reason we
  // allow share_write and delete(allows rename).
  HANDLE hFile = INVALID_HANDLE_VALUE;
D
Dmitri Smirnov 已提交
125 126 127 128 129 130 131

  DWORD fileFlags = FILE_ATTRIBUTE_READONLY;

  if (options.use_direct_reads && !options.use_mmap_reads) {
    fileFlags |= FILE_FLAG_NO_BUFFERING;
  }

132 133 134 135 136 137
  {
    IOSTATS_TIMER_GUARD(open_nanos);
    hFile = CreateFileA(
      fname.c_str(), GENERIC_READ,
      FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL,
      OPEN_EXISTING,  // Original fopen mode is "rb"
D
Dmitri Smirnov 已提交
138
      fileFlags, NULL);
139
  }
D
Dmitri Smirnov 已提交
140

141 142 143 144 145 146
  if (INVALID_HANDLE_VALUE == hFile) {
    auto lastError = GetLastError();
    s = IOErrorFromWindowsError("Failed to open NewSequentialFile" + fname,
      lastError);
  } else {
    result->reset(new WinSequentialFile(fname, hFile, options));
D
Dmitri Smirnov 已提交
147
  }
148 149
  return s;
}
D
Dmitri Smirnov 已提交
150

151 152 153 154 155
Status WinEnvIO::NewRandomAccessFile(const std::string& fname,
  std::unique_ptr<RandomAccessFile>* result,
  const EnvOptions& options) {
  result->reset();
  Status s;
D
Dmitri Smirnov 已提交
156

157 158 159
  // Open the file for read-only random access
  // Random access is to disable read-ahead as the system reads too much data
  DWORD fileFlags = FILE_ATTRIBUTE_READONLY;
D
Dmitri Smirnov 已提交
160

A
Aaron Gao 已提交
161
  if (options.use_direct_reads && !options.use_mmap_reads) {
162 163 164
    fileFlags |= FILE_FLAG_NO_BUFFERING;
  } else {
    fileFlags |= FILE_FLAG_RANDOM_ACCESS;
D
Dmitri Smirnov 已提交
165
  }
D
Dmitri Smirnov 已提交
166

167 168 169 170 171 172 173 174 175
  /// Shared access is necessary for corruption test to pass
  // almost all tests would work with a possible exception of fault_injection
  HANDLE hFile = 0;
  {
    IOSTATS_TIMER_GUARD(open_nanos);
    hFile =
      CreateFileA(fname.c_str(), GENERIC_READ,
      FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
      NULL, OPEN_EXISTING, fileFlags, NULL);
D
Dmitri Smirnov 已提交
176 177
  }

178 179 180 181
  if (INVALID_HANDLE_VALUE == hFile) {
    auto lastError = GetLastError();
    return IOErrorFromWindowsError(
      "NewRandomAccessFile failed to Create/Open: " + fname, lastError);
D
Dmitri Smirnov 已提交
182 183
  }

184
  UniqueCloseHandlePtr fileGuard(hFile, CloseHandleFunc);
D
Dmitri Smirnov 已提交
185

186 187 188 189
  // CAUTION! This will map the entire file into the process address space
  if (options.use_mmap_reads && sizeof(void*) >= 8) {
    // Use mmap when virtual address-space is plentiful.
    uint64_t fileSize;
D
Dmitri Smirnov 已提交
190

191
    s = GetFileSize(fname, &fileSize);
192

193 194 195 196 197
    if (s.ok()) {
      // Will not map empty files
      if (fileSize == 0) {
        return IOError(
          "NewRandomAccessFile failed to map empty file: " + fname, EINVAL);
D
Dmitri Smirnov 已提交
198
      }
199

200 201 202 203
      HANDLE hMap = CreateFileMappingA(hFile, NULL, PAGE_READONLY,
        0,  // Whole file at its present length
        0,
        NULL);  // Mapping name
D
Dmitri Smirnov 已提交
204

205 206 207 208 209
      if (!hMap) {
        auto lastError = GetLastError();
        return IOErrorFromWindowsError(
          "Failed to create file mapping for NewRandomAccessFile: " + fname,
          lastError);
D
Dmitri Smirnov 已提交
210 211
      }

212
      UniqueCloseHandlePtr mapGuard(hMap, CloseHandleFunc);
D
Dmitri Smirnov 已提交
213

214 215 216 217 218 219
      const void* mapped_region =
        MapViewOfFileEx(hMap, FILE_MAP_READ,
        0,  // High DWORD of access start
        0,  // Low DWORD
        fileSize,
        NULL);  // Let the OS choose the mapping
D
Dmitri Smirnov 已提交
220

221 222
      if (!mapped_region) {
        auto lastError = GetLastError();
S
sdong 已提交
223
        return IOErrorFromWindowsError(
224 225
          "Failed to MapViewOfFile for NewRandomAccessFile: " + fname,
          lastError);
S
sdong 已提交
226
      }
227

228 229 230 231 232
      result->reset(new WinMmapReadableFile(fname, hFile, hMap, mapped_region,
        fileSize));

      mapGuard.release();
      fileGuard.release();
D
Dmitri Smirnov 已提交
233
    }
234 235 236 237 238 239
  } else {
    result->reset(new WinRandomAccessFile(fname, hFile, page_size_, options));
    fileGuard.release();
  }
  return s;
}
D
Dmitri Smirnov 已提交
240

241 242 243 244 245
Status WinEnvIO::OpenWritableFile(const std::string& fname,
  std::unique_ptr<WritableFile>* result,
  const EnvOptions& options,
  bool reopen) {

246
  const size_t c_BufferCapacity = 64 * 1024;
D
Dmitri Smirnov 已提交
247

248
  EnvOptions local_options(options);
D
Dmitri Smirnov 已提交
249

250 251
  result->reset();
  Status s;
D
Dmitri Smirnov 已提交
252

253
  DWORD fileFlags = FILE_ATTRIBUTE_NORMAL;
D
Dmitri Smirnov 已提交
254

A
Aaron Gao 已提交
255
  if (local_options.use_direct_writes && !local_options.use_mmap_writes) {
256
    fileFlags = FILE_FLAG_NO_BUFFERING;
D
Dmitri Smirnov 已提交
257 258
  }

259 260 261 262 263 264 265
  // Desired access. We are want to write only here but if we want to memory
  // map
  // the file then there is no write only mode so we have to create it
  // Read/Write
  // However, MapViewOfFile specifies only Write only
  DWORD desired_access = GENERIC_WRITE;
  DWORD shared_mode = FILE_SHARE_READ;
D
Dmitri Smirnov 已提交
266

267 268
  if (local_options.use_mmap_writes) {
    desired_access |= GENERIC_READ;
269 270
  }
  else {
271 272 273 274
    // Adding this solely for tests to pass (fault_injection_test,
    // wal_manager_test).
    shared_mode |= (FILE_SHARE_WRITE | FILE_SHARE_DELETE);
  }
D
Dmitri Smirnov 已提交
275

276 277 278 279 280 281
  // This will always truncate the file
  DWORD creation_disposition = CREATE_ALWAYS;
  if (reopen) {
    creation_disposition = OPEN_ALWAYS;
  }

282 283 284 285 286 287 288 289
  HANDLE hFile = 0;
  {
    IOSTATS_TIMER_GUARD(open_nanos);
    hFile = CreateFileA(
      fname.c_str(),
      desired_access,  // Access desired
      shared_mode,
      NULL,           // Security attributes
290
      creation_disposition,  // Posix env says (reopen) ? (O_CREATE | O_APPEND) : O_CREAT | O_TRUNC
291 292 293 294 295 296 297 298
      fileFlags,      // Flags
      NULL);          // Template File
  }

  if (INVALID_HANDLE_VALUE == hFile) {
    auto lastError = GetLastError();
    return IOErrorFromWindowsError(
      "Failed to create a NewWriteableFile: " + fname, lastError);
D
Dmitri Smirnov 已提交
299 300
  }

301 302 303 304 305 306 307 308 309 310 311 312
  // We will start writing at the end, appending
  if (reopen) {
    LARGE_INTEGER zero_move;
    zero_move.QuadPart = 0;
    BOOL ret = SetFilePointerEx(hFile, zero_move, NULL, FILE_END);
    if (!ret) {
      auto lastError = GetLastError();
      return IOErrorFromWindowsError(
        "Failed to create a ReopenWritableFile move to the end: " + fname, lastError);
    }
  }

313 314 315 316 317 318 319 320 321 322
  if (options.use_mmap_writes) {
    // We usually do not use mmmapping on SSD and thus we pass memory
    // page_size
    result->reset(new WinMmapFile(fname, hFile, page_size_,
      allocation_granularity_, local_options));
  } else {
    // Here we want the buffer allocation to be aligned by the SSD page size
    // and to be a multiple of it
    result->reset(new WinWritableFile(fname, hFile, page_size_,
      c_BufferCapacity, local_options));
323
  }
324 325
  return s;
}
326

327
Status WinEnvIO::NewRandomRWFile(const std::string & fname,
328
  std::unique_ptr<RandomRWFile>* result, const EnvOptions & options) {
329 330 331 332 333 334 335 336 337 338

  Status s;

  // Open the file for read-only random access
  // Random access is to disable read-ahead as the system reads too much data
  DWORD desired_access = GENERIC_READ | GENERIC_WRITE;
  DWORD shared_mode = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE;
  DWORD creation_disposition = OPEN_ALWAYS; // Create if necessary or open existing
  DWORD file_flags = FILE_FLAG_RANDOM_ACCESS;

A
Aaron Gao 已提交
339
  if (options.use_direct_reads && options.use_direct_writes) {
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    file_flags |= FILE_FLAG_NO_BUFFERING;
  }

  /// Shared access is necessary for corruption test to pass
  // almost all tests would work with a possible exception of fault_injection
  HANDLE hFile = 0;
  {
    IOSTATS_TIMER_GUARD(open_nanos);
    hFile =
      CreateFileA(fname.c_str(),
        desired_access,
        shared_mode,
        NULL, // Security attributes
        creation_disposition,
        file_flags,
        NULL);
  }

  if (INVALID_HANDLE_VALUE == hFile) {
    auto lastError = GetLastError();
    return IOErrorFromWindowsError(
      "NewRandomRWFile failed to Create/Open: " + fname, lastError);
  }

  UniqueCloseHandlePtr fileGuard(hFile, CloseHandleFunc);
  result->reset(new WinRandomRWFile(fname, hFile, page_size_, options));
  fileGuard.release();

  return s;
}

371 372 373 374 375 376 377 378 379 380 381 382 383 384
Status WinEnvIO::NewDirectory(const std::string& name,
  std::unique_ptr<Directory>* result) {
  Status s;
  // Must be nullptr on failure
  result->reset();
  // Must fail if directory does not exist
  if (!DirExists(name)) {
    s = IOError("Directory does not exist: " + name, EEXIST);
  } else {
    IOSTATS_TIMER_GUARD(open_nanos);
    result->reset(new WinDirectory);
  }
  return s;
}
D
Dmitri Smirnov 已提交
385

386 387 388 389 390 391
Status WinEnvIO::FileExists(const std::string& fname) {
  // F_OK == 0
  const int F_OK_ = 0;
  return _access(fname.c_str(), F_OK_) == 0 ? Status::OK()
    : Status::NotFound();
}
D
Dmitri Smirnov 已提交
392

393 394
Status WinEnvIO::GetChildren(const std::string& dir,
  std::vector<std::string>* result) {
D
Dmitri Smirnov 已提交
395 396

  result->clear();
397
  std::vector<std::string> output;
D
Dmitri Smirnov 已提交
398

399
  Status status;
D
Dmitri Smirnov 已提交
400

401 402 403
  auto CloseDir = [](DIR* p) { closedir(p); };
  std::unique_ptr<DIR, decltype(CloseDir)> dirp(opendir(dir.c_str()),
    CloseDir);
D
Dmitri Smirnov 已提交
404

405
  if (!dirp) {
406 407 408 409 410 411 412 413
    switch (errno) {
      case EACCES:
      case ENOENT:
      case ENOTDIR:
        return Status::NotFound();
      default:
        return IOError(dir, errno);
    }
414 415 416
  } else {
    if (result->capacity() > 0) {
      output.reserve(result->capacity());
D
Dmitri Smirnov 已提交
417 418
    }

419 420 421 422 423 424
    struct dirent* ent = readdir(dirp.get());
    while (ent) {
      output.push_back(ent->d_name);
      ent = readdir(dirp.get());
    }
  }
425

426
  output.swap(*result);
D
Dmitri Smirnov 已提交
427

428 429
  return status;
}
D
Dmitri Smirnov 已提交
430

431 432
Status WinEnvIO::CreateDir(const std::string& name) {
  Status result;
D
Dmitri Smirnov 已提交
433

434 435 436
  if (_mkdir(name.c_str()) != 0) {
    auto code = errno;
    result = IOError("Failed to create dir: " + name, code);
D
Dmitri Smirnov 已提交
437 438
  }

439 440
  return result;
}
D
Dmitri Smirnov 已提交
441

442 443 444 445 446
Status  WinEnvIO::CreateDirIfMissing(const std::string& name) {
  Status result;

  if (DirExists(name)) {
    return result;
D
Dmitri Smirnov 已提交
447 448
  }

449 450 451 452 453 454 455
  if (_mkdir(name.c_str()) != 0) {
    if (errno == EEXIST) {
      result =
        Status::IOError("`" + name + "' exists but is not a directory");
    } else {
      auto code = errno;
      result = IOError("Failed to create dir: " + name, code);
D
Dmitri Smirnov 已提交
456 457 458
    }
  }

459 460
  return result;
}
D
Dmitri Smirnov 已提交
461

462 463 464 465 466
Status WinEnvIO::DeleteDir(const std::string& name) {
  Status result;
  if (_rmdir(name.c_str()) != 0) {
    auto code = errno;
    result = IOError("Failed to remove dir: " + name, code);
D
Dmitri Smirnov 已提交
467
  }
468 469
  return result;
}
D
Dmitri Smirnov 已提交
470

471 472 473
Status WinEnvIO::GetFileSize(const std::string& fname,
  uint64_t* size) {
  Status s;
474

475 476 477 478 479 480 481 482 483
  WIN32_FILE_ATTRIBUTE_DATA attrs;
  if (GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, &attrs)) {
    ULARGE_INTEGER file_size;
    file_size.HighPart = attrs.nFileSizeHigh;
    file_size.LowPart = attrs.nFileSizeLow;
    *size = file_size.QuadPart;
  } else {
    auto lastError = GetLastError();
    s = IOErrorFromWindowsError("Can not get size for: " + fname, lastError);
D
Dmitri Smirnov 已提交
484
  }
485 486
  return s;
}
D
Dmitri Smirnov 已提交
487

488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
uint64_t WinEnvIO::FileTimeToUnixTime(const FILETIME& ftTime) {
  const uint64_t c_FileTimePerSecond = 10000000U;
  // UNIX epoch starts on 1970-01-01T00:00:00Z
  // Windows FILETIME starts on 1601-01-01T00:00:00Z
  // Therefore, we need to subtract the below number of seconds from
  // the seconds that we obtain from FILETIME with an obvious loss of
  // precision
  const uint64_t c_SecondBeforeUnixEpoch = 11644473600U;

  ULARGE_INTEGER li;
  li.HighPart = ftTime.dwHighDateTime;
  li.LowPart = ftTime.dwLowDateTime;

  uint64_t result =
    (li.QuadPart / c_FileTimePerSecond) - c_SecondBeforeUnixEpoch;
  return result;
}

Status WinEnvIO::GetFileModificationTime(const std::string& fname,
  uint64_t* file_mtime) {
  Status s;

  WIN32_FILE_ATTRIBUTE_DATA attrs;
  if (GetFileAttributesExA(fname.c_str(), GetFileExInfoStandard, &attrs)) {
    *file_mtime = FileTimeToUnixTime(attrs.ftLastWriteTime);
  } else {
    auto lastError = GetLastError();
    s = IOErrorFromWindowsError(
      "Can not get file modification time for: " + fname, lastError);
    *file_mtime = 0;
D
Dmitri Smirnov 已提交
518 519
  }

520 521
  return s;
}
D
Dmitri Smirnov 已提交
522

523 524 525
Status WinEnvIO::RenameFile(const std::string& src,
  const std::string& target) {
  Status result;
526

527 528 529 530
  // rename() is not capable of replacing the existing file as on Linux
  // so use OS API directly
  if (!MoveFileExA(src.c_str(), target.c_str(), MOVEFILE_REPLACE_EXISTING)) {
    DWORD lastError = GetLastError();
D
Dmitri Smirnov 已提交
531

532 533
    std::string text("Failed to rename: ");
    text.append(src).append(" to: ").append(target);
D
Dmitri Smirnov 已提交
534

535
    result = IOErrorFromWindowsError(text, lastError);
D
Dmitri Smirnov 已提交
536 537
  }

538 539
  return result;
}
540

541 542 543
Status WinEnvIO::LinkFile(const std::string& src,
  const std::string& target) {
  Status result;
D
Dmitri Smirnov 已提交
544

545 546
  if (!CreateHardLinkA(target.c_str(), src.c_str(), NULL)) {
    DWORD lastError = GetLastError();
547

548 549
    std::string text("Failed to link: ");
    text.append(src).append(" to: ").append(target);
550

551
    result = IOErrorFromWindowsError(text, lastError);
552 553
  }

554 555
  return result;
}
556

557 558 559
Status  WinEnvIO::LockFile(const std::string& lockFname,
  FileLock** lock) {
  assert(lock != nullptr);
560

561 562
  *lock = NULL;
  Status result;
563

564 565
  // No-sharing, this is a LOCK file
  const DWORD ExclusiveAccessON = 0;
566

567 568 569 570 571 572 573 574 575
  // Obtain exclusive access to the LOCK file
  // Previously, instead of NORMAL attr we set DELETE on close and that worked
  // well except with fault_injection test that insists on deleting it.
  HANDLE hFile = 0;
  {
    IOSTATS_TIMER_GUARD(open_nanos);
    hFile = CreateFileA(lockFname.c_str(), (GENERIC_READ | GENERIC_WRITE),
      ExclusiveAccessON, NULL, CREATE_ALWAYS,
      FILE_ATTRIBUTE_NORMAL, NULL);
576 577
  }

578 579 580 581 582 583
  if (INVALID_HANDLE_VALUE == hFile) {
    auto lastError = GetLastError();
    result = IOErrorFromWindowsError(
      "Failed to create lock file: " + lockFname, lastError);
  } else {
    *lock = new WinFileLock(hFile);
D
Dmitri Smirnov 已提交
584
  }
S
sdong 已提交
585

586 587
  return result;
}
D
Dmitri Smirnov 已提交
588

589 590
Status WinEnvIO::UnlockFile(FileLock* lock) {
  Status result;
591

592
  assert(lock != nullptr);
593

594
  delete lock;
D
Dmitri Smirnov 已提交
595

596 597
  return result;
}
598

599 600
Status WinEnvIO::GetTestDirectory(std::string* result) {
  std::string output;
D
Dmitri Smirnov 已提交
601

602 603 604 605 606 607
  const char* env = getenv("TEST_TMPDIR");
  if (env && env[0] != '\0') {
    output = env;
    CreateDir(output);
  } else {
    env = getenv("TMP");
D
Dmitri Smirnov 已提交
608

609 610 611 612
    if (env && env[0] != '\0') {
      output = env;
    } else {
      output = "c:\\tmp";
D
Dmitri Smirnov 已提交
613 614
    }

615
    CreateDir(output);
616 617
  }

618 619
  output.append("\\testrocksdb-");
  output.append(std::to_string(_getpid()));
620

621
  CreateDir(output);
D
Dmitri Smirnov 已提交
622

623
  output.swap(*result);
D
Dmitri Smirnov 已提交
624

625 626
  return Status::OK();
}
D
Dmitri Smirnov 已提交
627

628 629 630
Status WinEnvIO::NewLogger(const std::string& fname,
  std::shared_ptr<Logger>* result) {
  Status s;
D
Dmitri Smirnov 已提交
631

632
  result->reset();
D
Dmitri Smirnov 已提交
633

634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666
  HANDLE hFile = 0;
  {
    IOSTATS_TIMER_GUARD(open_nanos);
    hFile = CreateFileA(
      fname.c_str(), GENERIC_WRITE,
      FILE_SHARE_READ | FILE_SHARE_DELETE,  // In RocksDb log files are
      // renamed and deleted before
      // they are closed. This enables
      // doing so.
      NULL,
      CREATE_ALWAYS,  // Original fopen mode is "w"
      FILE_ATTRIBUTE_NORMAL, NULL);
  }

  if (INVALID_HANDLE_VALUE == hFile) {
    auto lastError = GetLastError();
    s = IOErrorFromWindowsError("Failed to open LogFile" + fname, lastError);
  } else {
    {
      // With log files we want to set the true creation time as of now
      // because the system
      // for some reason caches the attributes of the previous file that just
      // been renamed from
      // this name so auto_roll_logger_test fails
      FILETIME ft;
      GetSystemTimeAsFileTime(&ft);
      // Set creation, last access and last write time to the same value
      SetFileTime(hFile, &ft, &ft, &ft);
    }
    result->reset(new WinLogger(&WinEnvThreads::gettid, hosted_env_, hFile));
  }
  return s;
}
D
Dmitri Smirnov 已提交
667

668
uint64_t WinEnvIO::NowMicros() {
D
Dmitri Smirnov 已提交
669

670 671 672
  if (GetSystemTimePreciseAsFileTime_ != NULL) {
    // all std::chrono clocks on windows proved to return
    // values that may repeat that is not good enough for some uses.
O
Orgad Shaneh 已提交
673
    const int64_t c_UnixEpochStartTicks = 116444736000000000LL;
674
    const int64_t c_FtToMicroSec = 10;
D
Dmitri Smirnov 已提交
675

676 677 678 679 680
    // This interface needs to return system time and not
    // just any microseconds because it is often used as an argument
    // to TimedWait() on condition variable
    FILETIME ftSystemTime;
    GetSystemTimePreciseAsFileTime_(&ftSystemTime);
D
Dmitri Smirnov 已提交
681

682 683 684 685 686 687 688 689
    LARGE_INTEGER li;
    li.LowPart = ftSystemTime.dwLowDateTime;
    li.HighPart = ftSystemTime.dwHighDateTime;
    // Subtract unix epoch start
    li.QuadPart -= c_UnixEpochStartTicks;
    // Convert to microsecs
    li.QuadPart /= c_FtToMicroSec;
    return li.QuadPart;
690
  }
691 692 693
  using namespace std::chrono;
  return duration_cast<microseconds>(system_clock::now().time_since_epoch()).count();
}
D
Dmitri Smirnov 已提交
694

695 696 697 698 699 700 701 702 703 704 705 706 707
uint64_t WinEnvIO::NowNanos() {
  // all std::chrono clocks on windows have the same resolution that is only
  // good enough for microseconds but not nanoseconds
  // On Windows 8 and Windows 2012 Server
  // GetSystemTimePreciseAsFileTime(&current_time) can be used
  LARGE_INTEGER li;
  QueryPerformanceCounter(&li);
  // Convert to nanoseconds first to avoid loss of precision
  // and divide by frequency
  li.QuadPart *= std::nano::den;
  li.QuadPart /= perf_counter_frequency_;
  return li.QuadPart;
}
D
Dmitri Smirnov 已提交
708

709 710 711 712
Status WinEnvIO::GetHostName(char* name, uint64_t len) {
  Status s;
  DWORD nSize = static_cast<DWORD>(
    std::min<uint64_t>(len, std::numeric_limits<DWORD>::max()));
D
Dmitri Smirnov 已提交
713

714 715 716 717 718
  if (!::GetComputerNameA(name, &nSize)) {
    auto lastError = GetLastError();
    s = IOErrorFromWindowsError("GetHostName", lastError);
  } else {
    name[nSize] = 0;
719
  }
D
Dmitri Smirnov 已提交
720

721 722 723 724 725 726 727 728 729 730 731 732 733
  return s;
}

Status WinEnvIO::GetAbsolutePath(const std::string& db_path,
  std::string* output_path) {
  // Check if we already have an absolute path
  // that starts with non dot and has a semicolon in it
  if ((!db_path.empty() && (db_path[0] == '/' || db_path[0] == '\\')) ||
    (db_path.size() > 2 && db_path[0] != '.' &&
    ((db_path[1] == ':' && db_path[2] == '\\') ||
    (db_path[1] == ':' && db_path[2] == '/')))) {
    *output_path = db_path;
    return Status::OK();
D
Dmitri Smirnov 已提交
734 735
  }

736 737
  std::string result;
  result.resize(_MAX_PATH);
D
Dmitri Smirnov 已提交
738

739 740 741 742 743
  char* ret = _getcwd(&result[0], _MAX_PATH);
  if (ret == nullptr) {
    return Status::IOError("Failed to get current working directory",
      strerror(errno));
  }
D
Dmitri Smirnov 已提交
744

745
  result.resize(strlen(result.data()));
D
Dmitri Smirnov 已提交
746

747 748 749
  result.swap(*output_path);
  return Status::OK();
}
D
Dmitri Smirnov 已提交
750

751 752
std::string WinEnvIO::TimeToString(uint64_t secondsSince1970) {
  std::string result;
D
Dmitri Smirnov 已提交
753

754 755
  const time_t seconds = secondsSince1970;
  const int maxsize = 64;
D
Dmitri Smirnov 已提交
756

757 758
  struct tm t;
  errno_t ret = localtime_s(&t, &seconds);
D
Dmitri Smirnov 已提交
759

760 761 762 763 764
  if (ret) {
    result = std::to_string(seconds);
  } else {
    result.resize(maxsize);
    char* p = &result[0];
D
Dmitri Smirnov 已提交
765

766 767 768 769
    int len = snprintf(p, maxsize, "%04d/%02d/%02d-%02d:%02d:%02d ",
      t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour,
      t.tm_min, t.tm_sec);
    assert(len > 0);
D
Dmitri Smirnov 已提交
770

771
    result.resize(len);
D
Dmitri Smirnov 已提交
772 773
  }

774 775
  return result;
}
D
Dmitri Smirnov 已提交
776

777 778 779 780
EnvOptions WinEnvIO::OptimizeForLogWrite(const EnvOptions& env_options,
  const DBOptions& db_options) const {
  EnvOptions optimized = env_options;
  optimized.bytes_per_sync = db_options.wal_bytes_per_sync;
A
Aaron Gao 已提交
781 782
  optimized.use_mmap_writes = false;
  // This is because we flush only whole pages on unbuffered io and
783
  // the last records are not guaranteed to be flushed.
A
Aaron Gao 已提交
784
  optimized.use_direct_writes = false;
785 786 787 788 789 790
  // TODO(icanadi) it's faster if fallocate_with_keep_size is false, but it
  // breaks TransactionLogIteratorStallAtLastRecord unit test. Fix the unit
  // test and make this false
  optimized.fallocate_with_keep_size = true;
  return optimized;
}
D
Dmitri Smirnov 已提交
791

792 793 794 795
EnvOptions WinEnvIO::OptimizeForManifestWrite(
  const EnvOptions& env_options) const {
  EnvOptions optimized = env_options;
  optimized.use_mmap_writes = false;
A
Aaron Gao 已提交
796
  optimized.use_direct_writes = false;
797 798 799 800 801 802 803 804 805
  optimized.fallocate_with_keep_size = true;
  return optimized;
}

// Returns true iff the named directory exists and is a directory.
bool WinEnvIO::DirExists(const std::string& dname) {
  WIN32_FILE_ATTRIBUTE_DATA attrs;
  if (GetFileAttributesExA(dname.c_str(), GetFileExInfoStandard, &attrs)) {
    return 0 != (attrs.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
D
Dmitri Smirnov 已提交
806
  }
807 808
  return false;
}
D
Dmitri Smirnov 已提交
809

810 811
////////////////////////////////////////////////////////////////////////
// WinEnvThreads
D
Dmitri Smirnov 已提交
812

813
WinEnvThreads::WinEnvThreads(Env* hosted_env) : hosted_env_(hosted_env), thread_pools_(Env::Priority::TOTAL) {
D
Dmitri Smirnov 已提交
814

815 816 817 818 819
  for (int pool_id = 0; pool_id < Env::Priority::TOTAL; ++pool_id) {
    thread_pools_[pool_id].SetThreadPriority(
      static_cast<Env::Priority>(pool_id));
    // This allows later initializing the thread-local-env of each thread.
    thread_pools_[pool_id].SetHostEnv(hosted_env);
D
Dmitri Smirnov 已提交
820
  }
821
}
D
Dmitri Smirnov 已提交
822

823
WinEnvThreads::~WinEnvThreads() {
D
Dmitri Smirnov 已提交
824

825
  WaitForJoin();
D
Dmitri Smirnov 已提交
826

827 828
  for (auto& thpool : thread_pools_) {
    thpool.JoinAllThreads();
D
Dmitri Smirnov 已提交
829 830 831
  }
}

832 833
void WinEnvThreads::Schedule(void(*function)(void*), void* arg, Env::Priority pri,
  void* tag, void(*unschedFunction)(void* arg)) {
A
Andrew Kryczka 已提交
834
  assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH);
835 836
  thread_pools_[pri].Schedule(function, arg, tag, unschedFunction);
}
D
Dmitri Smirnov 已提交
837

838 839 840
int WinEnvThreads::UnSchedule(void* arg, Env::Priority pri) {
  return thread_pools_[pri].UnSchedule(arg);
}
D
Dmitri Smirnov 已提交
841

842
namespace {
D
Dmitri Smirnov 已提交
843

844 845 846 847
  struct StartThreadState {
    void(*user_function)(void*);
    void* arg;
  };
D
Dmitri Smirnov 已提交
848

849 850 851 852 853
  void* StartThreadWrapper(void* arg) {
    std::unique_ptr<StartThreadState> state(
      reinterpret_cast<StartThreadState*>(arg));
    state->user_function(state->arg);
    return nullptr;
D
Dmitri Smirnov 已提交
854 855
  }

856
}
D
Dmitri Smirnov 已提交
857

858 859 860 861 862
void WinEnvThreads::StartThread(void(*function)(void* arg), void* arg) {
  std::unique_ptr<StartThreadState> state(new StartThreadState);
  state->user_function = function;
  state->arg = arg;
  try {
D
Dmitri Smirnov 已提交
863

D
Dmitri Smirnov 已提交
864
    rocksdb::port::WindowsThread th(&StartThreadWrapper, state.get());
865
    state.release();
D
Dmitri Smirnov 已提交
866

867 868
    std::lock_guard<std::mutex> lg(mu_);
    threads_to_join_.push_back(std::move(th));
D
Dmitri Smirnov 已提交
869

870 871
  } catch (const std::system_error& ex) {
    WinthreadCall("start thread", ex.code());
D
Dmitri Smirnov 已提交
872
  }
873
}
D
Dmitri Smirnov 已提交
874

875 876 877
void WinEnvThreads::WaitForJoin() {
  for (auto& th : threads_to_join_) {
    th.join();
D
Dmitri Smirnov 已提交
878
  }
879 880
  threads_to_join_.clear();
}
D
Dmitri Smirnov 已提交
881

882
unsigned int WinEnvThreads::GetThreadPoolQueueLen(Env::Priority pri) const {
A
Andrew Kryczka 已提交
883
  assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH);
884 885
  return thread_pools_[pri].GetQueueLen();
}
D
Dmitri Smirnov 已提交
886

887 888 889 890
uint64_t WinEnvThreads::gettid() {
  uint64_t thread_id = GetCurrentThreadId();
  return thread_id;
}
D
Dmitri Smirnov 已提交
891

892
uint64_t WinEnvThreads::GetThreadID() const { return gettid(); }
D
Dmitri Smirnov 已提交
893

894 895 896
void  WinEnvThreads::SleepForMicroseconds(int micros) {
  std::this_thread::sleep_for(std::chrono::microseconds(micros));
}
D
Dmitri Smirnov 已提交
897

898
void WinEnvThreads::SetBackgroundThreads(int num, Env::Priority pri) {
A
Andrew Kryczka 已提交
899
  assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH);
900 901
  thread_pools_[pri].SetBackgroundThreads(num);
}
D
Dmitri Smirnov 已提交
902

903
int WinEnvThreads::GetBackgroundThreads(Env::Priority pri) {
A
Andrew Kryczka 已提交
904
  assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH);
905 906 907
  return thread_pools_[pri].GetBackgroundThreads();
}

908
void WinEnvThreads::IncBackgroundThreadsIfNeeded(int num, Env::Priority pri) {
A
Andrew Kryczka 已提交
909
  assert(pri >= Env::Priority::BOTTOM && pri <= Env::Priority::HIGH);
910 911
  thread_pools_[pri].IncBackgroundThreadsIfNeeded(num);
}
D
Dmitri Smirnov 已提交
912

913 914
/////////////////////////////////////////////////////////////////////////
// WinEnv
D
Dmitri Smirnov 已提交
915

916 917 918 919
WinEnv::WinEnv() : winenv_io_(this), winenv_threads_(this) {
  // Protected member of the base class
  thread_status_updater_ = CreateThreadStatusUpdater();
}
D
Dmitri Smirnov 已提交
920 921


922 923 924 925 926
WinEnv::~WinEnv() {
  // All threads must be joined before the deletion of
  // thread_status_updater_.
  delete thread_status_updater_;
}
D
Dmitri Smirnov 已提交
927

928 929 930 931 932
Status WinEnv::GetThreadList(
  std::vector<ThreadStatus>* thread_list) {
  assert(thread_status_updater_);
  return thread_status_updater_->GetThreadList(thread_list);
}
D
Dmitri Smirnov 已提交
933

934 935 936
Status WinEnv::DeleteFile(const std::string& fname) {
  return winenv_io_.DeleteFile(fname);
}
D
Dmitri Smirnov 已提交
937

938 939 940
Status WinEnv::GetCurrentTime(int64_t* unix_time) {
  return winenv_io_.GetCurrentTime(unix_time);
}
D
Dmitri Smirnov 已提交
941

942 943 944 945 946
Status  WinEnv::NewSequentialFile(const std::string& fname,
  std::unique_ptr<SequentialFile>* result,
  const EnvOptions& options) {
  return winenv_io_.NewSequentialFile(fname, result, options);
}
D
Dmitri Smirnov 已提交
947

948 949 950 951 952
Status WinEnv::NewRandomAccessFile(const std::string& fname,
  std::unique_ptr<RandomAccessFile>* result,
  const EnvOptions& options) {
  return winenv_io_.NewRandomAccessFile(fname, result, options);
}
D
Dmitri Smirnov 已提交
953

954
Status WinEnv::NewWritableFile(const std::string& fname,
A
Aaron Gao 已提交
955 956
                               std::unique_ptr<WritableFile>* result,
                               const EnvOptions& options) {
957 958 959 960 961 962
  return winenv_io_.OpenWritableFile(fname, result, options, false);
}

Status WinEnv::ReopenWritableFile(const std::string& fname,
    std::unique_ptr<WritableFile>* result, const EnvOptions& options) {
  return winenv_io_.OpenWritableFile(fname, result, options, true);
963
}
D
Dmitri Smirnov 已提交
964

965 966 967 968 969
Status WinEnv::NewRandomRWFile(const std::string & fname,
  unique_ptr<RandomRWFile>* result, const EnvOptions & options) {
  return winenv_io_.NewRandomRWFile(fname, result, options);
}

970 971 972 973
Status WinEnv::NewDirectory(const std::string& name,
  std::unique_ptr<Directory>* result) {
  return winenv_io_.NewDirectory(name, result);
}
D
Dmitri Smirnov 已提交
974

975 976 977
Status WinEnv::FileExists(const std::string& fname) {
  return winenv_io_.FileExists(fname);
}
D
Dmitri Smirnov 已提交
978

979 980 981 982
Status WinEnv::GetChildren(const std::string& dir,
  std::vector<std::string>* result) {
  return winenv_io_.GetChildren(dir, result);
}
D
Dmitri Smirnov 已提交
983

984 985 986
Status WinEnv::CreateDir(const std::string& name) {
  return winenv_io_.CreateDir(name);
}
D
Dmitri Smirnov 已提交
987

988 989 990
Status WinEnv::CreateDirIfMissing(const std::string& name) {
  return winenv_io_.CreateDirIfMissing(name);
}
D
Dmitri Smirnov 已提交
991

992 993 994
Status WinEnv::DeleteDir(const std::string& name) {
  return winenv_io_.DeleteDir(name);
}
D
Dmitri Smirnov 已提交
995

996 997 998 999
Status WinEnv::GetFileSize(const std::string& fname,
  uint64_t* size) {
  return winenv_io_.GetFileSize(fname, size);
}
D
Dmitri Smirnov 已提交
1000

1001 1002 1003 1004
Status  WinEnv::GetFileModificationTime(const std::string& fname,
  uint64_t* file_mtime) {
  return winenv_io_.GetFileModificationTime(fname, file_mtime);
}
D
Dmitri Smirnov 已提交
1005

1006 1007 1008 1009
Status WinEnv::RenameFile(const std::string& src,
  const std::string& target) {
  return winenv_io_.RenameFile(src, target);
}
D
Dmitri Smirnov 已提交
1010

1011 1012 1013 1014
Status WinEnv::LinkFile(const std::string& src,
  const std::string& target) {
  return winenv_io_.LinkFile(src, target);
}
D
Dmitri Smirnov 已提交
1015

1016 1017 1018 1019
Status WinEnv::LockFile(const std::string& lockFname,
  FileLock** lock) {
  return winenv_io_.LockFile(lockFname, lock);
}
D
Dmitri Smirnov 已提交
1020

1021 1022 1023
Status WinEnv::UnlockFile(FileLock* lock) {
  return winenv_io_.UnlockFile(lock);
}
D
Dmitri Smirnov 已提交
1024

1025 1026 1027
Status  WinEnv::GetTestDirectory(std::string* result) {
  return winenv_io_.GetTestDirectory(result);
}
D
Dmitri Smirnov 已提交
1028

1029 1030 1031 1032
Status WinEnv::NewLogger(const std::string& fname,
  std::shared_ptr<Logger>* result) {
  return winenv_io_.NewLogger(fname, result);
}
1033

1034 1035 1036
uint64_t WinEnv::NowMicros() {
  return winenv_io_.NowMicros();
}
1037

1038 1039 1040
uint64_t  WinEnv::NowNanos() {
  return winenv_io_.NowNanos();
}
D
Dmitri Smirnov 已提交
1041

1042 1043 1044
Status WinEnv::GetHostName(char* name, uint64_t len) {
  return winenv_io_.GetHostName(name, len);
}
D
Dmitri Smirnov 已提交
1045

1046 1047 1048 1049
Status WinEnv::GetAbsolutePath(const std::string& db_path,
  std::string* output_path) {
  return winenv_io_.GetAbsolutePath(db_path, output_path);
}
D
Dmitri Smirnov 已提交
1050

1051 1052 1053
std::string WinEnv::TimeToString(uint64_t secondsSince1970) {
  return winenv_io_.TimeToString(secondsSince1970);
}
D
Dmitri Smirnov 已提交
1054

1055 1056 1057 1058
void  WinEnv::Schedule(void(*function)(void*), void* arg, Env::Priority pri,
  void* tag,
  void(*unschedFunction)(void* arg)) {
  return winenv_threads_.Schedule(function, arg, pri, tag, unschedFunction);
D
Dmitri Smirnov 已提交
1059 1060
}

1061 1062
int WinEnv::UnSchedule(void* arg, Env::Priority pri) {
  return winenv_threads_.UnSchedule(arg, pri);
D
Dmitri Smirnov 已提交
1063 1064
}

1065 1066
void WinEnv::StartThread(void(*function)(void* arg), void* arg) {
  return winenv_threads_.StartThread(function, arg);
D
Dmitri Smirnov 已提交
1067 1068
}

1069 1070
void WinEnv::WaitForJoin() {
  return winenv_threads_.WaitForJoin();
D
Dmitri Smirnov 已提交
1071 1072
}

1073 1074
unsigned int  WinEnv::GetThreadPoolQueueLen(Env::Priority pri) const {
  return winenv_threads_.GetThreadPoolQueueLen(pri);
D
Dmitri Smirnov 已提交
1075 1076
}

1077 1078
uint64_t WinEnv::GetThreadID() const {
  return winenv_threads_.GetThreadID();
D
Dmitri Smirnov 已提交
1079 1080
}

1081 1082 1083
void WinEnv::SleepForMicroseconds(int micros) {
  return winenv_threads_.SleepForMicroseconds(micros);
}
D
Dmitri Smirnov 已提交
1084

1085 1086 1087 1088
// Allow increasing the number of worker threads.
void  WinEnv::SetBackgroundThreads(int num, Env::Priority pri) {
  return winenv_threads_.SetBackgroundThreads(num, pri);
}
D
Dmitri Smirnov 已提交
1089

1090 1091 1092 1093
int WinEnv::GetBackgroundThreads(Env::Priority pri) {
  return winenv_threads_.GetBackgroundThreads(pri);
}

1094 1095
void  WinEnv::IncBackgroundThreadsIfNeeded(int num, Env::Priority pri) {
  return winenv_threads_.IncBackgroundThreadsIfNeeded(num, pri);
D
Dmitri Smirnov 已提交
1096 1097
}

1098 1099 1100 1101
EnvOptions WinEnv::OptimizeForLogWrite(const EnvOptions& env_options,
  const DBOptions& db_options) const {
  return winenv_io_.OptimizeForLogWrite(env_options, db_options);
}
D
Dmitri Smirnov 已提交
1102

1103 1104 1105
EnvOptions WinEnv::OptimizeForManifestWrite(
  const EnvOptions& env_options) const {
  return winenv_io_.OptimizeForManifestWrite(env_options);
D
Dmitri Smirnov 已提交
1106 1107
}

1108
}  // namespace port
D
Dmitri Smirnov 已提交
1109 1110 1111 1112 1113 1114 1115 1116

std::string Env::GenerateUniqueId() {
  std::string result;

  UUID uuid;
  UuidCreateSequential(&uuid);

  RPC_CSTR rpc_str;
O
Orgad Shaneh 已提交
1117 1118 1119
  auto status = UuidToStringA(&uuid, &rpc_str);
  (void)status;
  assert(status == RPC_S_OK);
D
Dmitri Smirnov 已提交
1120 1121 1122

  result = reinterpret_cast<char*>(rpc_str);

O
Orgad Shaneh 已提交
1123 1124
  status = RpcStringFreeA(&rpc_str);
  assert(status == RPC_S_OK);
D
Dmitri Smirnov 已提交
1125 1126 1127 1128 1129

  return result;
}

}  // namespace rocksdb