osFile.c 12.5 KB
Newer Older
S
Shengliang Guan 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "os.h"

S
Shengliang Guan 已提交
18
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#include <io.h>

#if defined(_MSDOS)
#define open _open
#endif

#if defined(_WIN32)
extern int openA(const char *, int, ...); /* MsvcLibX ANSI version of open */
extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */
#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
#define open openU
#else /* _ANSI_SOURCE */
#define open openA
#endif /* defined(_UTF8_SOURCE) */
#endif /* defined(_WIN32) */

S
Shengliang Guan 已提交
35 36 37 38 39 40 41 42 43
#else
#include <fcntl.h>
#include <sys/file.h>
#include <sys/sendfile.h>
#include <sys/stat.h>
#include <unistd.h>
#endif

void taosCloseFile(FileFd fd) {
S
TD-4088  
Shengliang Guan 已提交
44 45 46 47
  close(fd);
  fd = FD_INITIALIZER;
}

S
Shengliang Guan 已提交
48
void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) {
S
TD-4088  
Shengliang Guan 已提交
49
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
S
Shengliang Guan 已提交
50
  const char *tdengineTmpFileNamePrefix = "tdengine-";
S
TD-4088  
Shengliang Guan 已提交
51
  char        tmpPath[PATH_MAX];
S
Shengliang Guan 已提交
52

S
Shengliang Guan 已提交
53 54
  int32_t len = (int32_t)strlen(inputTmpDir);
  memcpy(tmpPath, inputTmpDir, len);
S
TD-4088  
Shengliang Guan 已提交
55 56 57

  if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') {
    tmpPath[len++] = '\\';
58
  }
S
Shengliang Guan 已提交
59

60
  strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
S
TD-4088  
Shengliang Guan 已提交
61
  strcat(tmpPath, tdengineTmpFileNamePrefix);
S
Shengliang Guan 已提交
62 63 64
  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
    strcat(tmpPath, fileNamePrefix);
    strcat(tmpPath, "-%d-%s");
S
Shengliang Guan 已提交
65
  }
S
Shengliang Guan 已提交
66

S
TD-4088  
Shengliang Guan 已提交
67 68
  char rand[8] = {0};
  taosRandStr(rand, tListLen(rand) - 1);
S
Shengliang Guan 已提交
69
  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
S
TD-1912  
Shengliang Guan 已提交
70

S
TD-4088  
Shengliang Guan 已提交
71
#else
S
Shengliang Guan 已提交
72

S
TD-4088  
Shengliang Guan 已提交
73
  const char *tdengineTmpFileNamePrefix = "tdengine-";
S
Shengliang Guan 已提交
74

S
Shengliang Guan 已提交
75 76 77
  char tmpPath[PATH_MAX];
  int32_t len = strlen(inputTmpDir);
  memcpy(tmpPath, inputTmpDir, len);
S
TD-4088  
Shengliang Guan 已提交
78
  static uint64_t seqId = 0;
S
Shengliang Guan 已提交
79

S
TD-4088  
Shengliang Guan 已提交
80 81
  if (tmpPath[len - 1] != '/') {
    tmpPath[len++] = '/';
S
TD-1912  
Shengliang Guan 已提交
82
  }
S
Shengliang Guan 已提交
83

S
TD-4088  
Shengliang Guan 已提交
84 85 86 87
  strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
    strcat(tmpPath, fileNamePrefix);
    strcat(tmpPath, "-%d-%s");
S
Shengliang Guan 已提交
88 89
  }

S
TD-4088  
Shengliang Guan 已提交
90 91 92
  char rand[32] = {0};

  sprintf(rand, "%" PRIu64, atomic_add_fetch_64(&seqId, 1));
S
Shengliang Guan 已提交
93

S
TD-4088  
Shengliang Guan 已提交
94
  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
S
Shengliang Guan 已提交
95

S
TD-4088  
Shengliang Guan 已提交
96
#endif
S
Shengliang Guan 已提交
97
}
S
TD-4088  
Shengliang Guan 已提交
98

S
Shengliang Guan 已提交
99
int64_t taosReadFile(FileFd fd, void *buf, int64_t count) {
S
TD-1912  
Shengliang Guan 已提交
100 101
  int64_t leftbytes = count;
  int64_t readbytes;
S
Shengliang Guan 已提交
102 103 104
  char *  tbuf = (char *)buf;

  while (leftbytes > 0) {
S
slguan 已提交
105
    readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes);
S
Shengliang Guan 已提交
106 107 108 109 110 111 112
    if (readbytes < 0) {
      if (errno == EINTR) {
        continue;
      } else {
        return -1;
      }
    } else if (readbytes == 0) {
S
TD-1912  
Shengliang Guan 已提交
113
      return (int64_t)(count - leftbytes);
S
Shengliang Guan 已提交
114 115 116 117 118 119
    }

    leftbytes -= readbytes;
    tbuf += readbytes;
  }

S
TD-1912  
Shengliang Guan 已提交
120
  return count;
S
Shengliang Guan 已提交
121 122
}

H
more  
Hongze Cheng 已提交
123
int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) {
S
TD-1912  
Shengliang Guan 已提交
124 125
  int64_t nleft = n;
  int64_t nwritten = 0;
S
Shengliang Guan 已提交
126 127 128
  char *  tbuf = (char *)buf;

  while (nleft > 0) {
S
slguan 已提交
129
    nwritten = write(fd, (void *)tbuf, (uint32_t)nleft);
S
Shengliang Guan 已提交
130 131 132 133 134 135 136 137 138 139
    if (nwritten < 0) {
      if (errno == EINTR) {
        continue;
      }
      return -1;
    }
    nleft -= nwritten;
    tbuf += nwritten;
  }

S
TD-1912  
Shengliang Guan 已提交
140
  return n;
S
Shengliang Guan 已提交
141 142
}

S
Shengliang Guan 已提交
143
int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); }
S
Shengliang Guan 已提交
144

S
Shengliang Guan 已提交
145 146 147 148
int64_t taosCopyFile(char *from, char *to) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
H
Hongze Cheng 已提交
149 150
  char    buffer[4096];
  int     fidto = -1, fidfrom = -1;
H
Hongze Cheng 已提交
151 152
  int64_t size = 0;
  int64_t bytes;
H
Hongze Cheng 已提交
153

S
Shengliang Guan 已提交
154
  fidfrom = open(from, O_RDONLY);
H
Hongze Cheng 已提交
155 156
  if (fidfrom < 0) goto _err;

S
Shengliang Guan 已提交
157
  fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755);
H
Hongze Cheng 已提交
158 159 160
  if (fidto < 0) goto _err;

  while (true) {
S
Shengliang Guan 已提交
161
    bytes = taosReadFile(fidfrom, buffer, sizeof(buffer));
H
Hongze Cheng 已提交
162 163 164 165 166
    if (bytes < 0) goto _err;
    if (bytes == 0) break;

    size += bytes;

S
Shengliang Guan 已提交
167
    if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err;
H
Hongze Cheng 已提交
168 169 170
    if (bytes < sizeof(buffer)) break;
  }

S
Shengliang Guan 已提交
171
  taosFsyncFile(fidto);
H
Hongze Cheng 已提交
172

S
Shengliang Guan 已提交
173 174
  taosCloseFile(fidfrom);
  taosCloseFile(fidto);
H
Hongze Cheng 已提交
175 176 177
  return size;

_err:
S
Shengliang Guan 已提交
178 179
  if (fidfrom >= 0) taosCloseFile(fidfrom);
  if (fidto >= 0) taosCloseFile(fidto);
H
Hongze Cheng 已提交
180
  remove(to);
H
Hongze Cheng 已提交
181
  return -1;
S
Shengliang Guan 已提交
182
#endif
H
Hongze Cheng 已提交
183 184
}

S
TD-4088  
Shengliang Guan 已提交
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)

#define _SEND_FILE_STEP_ 1000

int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
  fseek(in_file, (int32_t)(*offset), 0);
  int64_t writeLen = 0;
  uint8_t buffer[_SEND_FILE_STEP_] = {0};

  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
    size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
    if (rlen <= 0) {
      return writeLen;
    } else if (rlen < _SEND_FILE_STEP_) {
      fwrite(buffer, 1, rlen, out_file);
      return (int64_t)(writeLen + rlen);
    } else {
      fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
      writeLen += _SEND_FILE_STEP_;
    }
  }

  int64_t remain = count - writeLen;
  if (remain > 0) {
    size_t rlen = fread(buffer, 1, (size_t)remain, in_file);
    if (rlen <= 0) {
      return writeLen;
    } else {
      fwrite(buffer, 1, (size_t)remain, out_file);
      writeLen += remain;
    }
  }

  return writeLen;
}

int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) {
  if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0);

  int64_t writeLen = 0;
  uint8_t buffer[_SEND_FILE_STEP_] = {0};

  for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
    int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_);
    if (rlen <= 0) {
      return writeLen;
    } else if (rlen < _SEND_FILE_STEP_) {
      taosWriteSocket(dfd, buffer, rlen);
      return (int64_t)(writeLen + rlen);
    } else {
      taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_);
      writeLen += _SEND_FILE_STEP_;
    }
  }

  int64_t remain = count - writeLen;
  if (remain > 0) {
    int32_t rlen = read(sfd, buffer, (int32_t)remain);
    if (rlen <= 0) {
      return writeLen;
    } else {
      taosWriteSocket(sfd, buffer, (int32_t)remain);
      writeLen += remain;
    }
  }

  return writeLen;
}

#elif defined(_TD_DARWIN_64)
S
TD-1912  
Shengliang Guan 已提交
255

S
TD-4088  
Shengliang Guan 已提交
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
  int r = 0;
  if (offset) {
    r = fseek(in_file, *offset, SEEK_SET);
    if (r == -1) return -1;
  }
  off_t len = count;
  while (len > 0) {
    char  buf[1024 * 16];
    off_t n = sizeof(buf);
    if (len < n) n = len;
    size_t m = fread(buf, 1, n, in_file);
    if (m < n) {
      int e = ferror(in_file);
      if (e) return -1;
    }
    if (m == 0) break;
    if (m != fwrite(buf, 1, m, out_file)) {
      return -1;
    }
    len -= m;
  }
  return count - len;
}

int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) {
  int r = 0;
  if (offset) {
    r = lseek(sfd, *offset, SEEK_SET);
    if (r == -1) return -1;
  }
  off_t len = count;
  while (len > 0) {
    char  buf[1024 * 16];
    off_t n = sizeof(buf);
    if (len < n) n = len;
    size_t m = read(sfd, buf, n);
    if (m == -1) return -1;
    if (m == 0) break;
    size_t l = write(dfd, buf, m);
    if (l == -1) return -1;
    len -= l;
  }
  return count - len;
}

#else

int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) {
S
TD-1912  
Shengliang Guan 已提交
305 306
  int64_t leftbytes = size;
  int64_t sentbytes;
S
Shengliang Guan 已提交
307 308 309 310

  while (leftbytes > 0) {
    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
    if (sentbytes == -1) {
S
TD-1985  
Shengliang Guan 已提交
311
      if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
S
Shengliang Guan 已提交
312 313 314 315 316
        continue;
      } else {
        return -1;
      }
    } else if (sentbytes == 0) {
S
TD-1912  
Shengliang Guan 已提交
317
      return (int64_t)(size - leftbytes);
S
Shengliang Guan 已提交
318 319 320 321 322 323 324
    }

    leftbytes -= sentbytes;
  }

  return size;
}
S
TD-1912  
Shengliang Guan 已提交
325 326 327 328 329 330 331

int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) {
  return taosSendFile(fileno(outfile), fileno(infile), offset, size);
}

#endif

S
Shengliang Guan 已提交
332
int32_t taosFtruncateFile(FileFd fd, int64_t l_size) {
S
TD-4088  
Shengliang Guan 已提交
333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  if (fd < 0) {
    errno = EBADF;
    uError("%s\n", "fd arg was negative");
    return -1;
  }

  HANDLE h = (HANDLE)_get_osfhandle(fd);

  LARGE_INTEGER li_0;
  li_0.QuadPart = (int64_t)0;
  BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
  if (!cur) {
    uError("SetFilePointerEx Error getting current position in file.\n");
    return -1;
  }
S
TD-1912  
Shengliang Guan 已提交
349

S
TD-4088  
Shengliang Guan 已提交
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
  LARGE_INTEGER li_size;
  li_size.QuadPart = l_size;
  BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
  if (cur2 == 0) {
    int error = GetLastError();
    uError("SetFilePointerEx GetLastError is: %d\n", error);
    switch (error) {
      case ERROR_INVALID_HANDLE:
        errno = EBADF;
        break;
      default:
        errno = EIO;
        break;
    }
    return -1;
  }

  if (!SetEndOfFile(h)) {
    int error = GetLastError();
    uError("SetEndOfFile GetLastError is:%d", error);
    switch (error) {
      case ERROR_INVALID_HANDLE:
        errno = EBADF;
        break;
      default:
        errno = EIO;
        break;
    }
    return -1;
  }

  return 0;
S
Shengliang Guan 已提交
382 383 384
#else
  return ftruncate(fd, l_size);
#endif
S
TD-4088  
Shengliang Guan 已提交
385 386
}

S
Shengliang Guan 已提交
387 388
int32_t taosFsyncFile(FileFd fd) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
S
TD-4088  
Shengliang Guan 已提交
389 390 391 392 393 394 395 396 397
  if (fd < 0) {
    errno = EBADF;
    uError("%s\n", "fd arg was negative");
    return -1;
  }

  HANDLE h = (HANDLE)_get_osfhandle(fd);

  return FlushFileBuffers(h);
S
Shengliang Guan 已提交
398 399 400
#else
  return fsync(fd);
#endif
S
TD-4088  
Shengliang Guan 已提交
401 402
}

S
Shengliang Guan 已提交
403 404
int32_t taosRenameFile(char *oldName, char *newName) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
S
TD-4088  
Shengliang Guan 已提交
405 406
  int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED);
  if (code < 0) {
S
Shengliang Guan 已提交
407
    //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
S
Shengliang Guan 已提交
408 409 410 411 412 413
  }

  return code;
#else
  int32_t code = rename(oldName, newName);
  if (code < 0) {
S
Shengliang Guan 已提交
414
    //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
S
TD-4088  
Shengliang Guan 已提交
415 416 417
  }

  return code;
S
Shengliang Guan 已提交
418 419 420 421 422 423 424 425 426
#endif
}

int32_t taosLockFile(int32_t fd) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
  return (int32_t)flock(fd, LOCK_EX | LOCK_NB);
#endif
S
TD-4088  
Shengliang Guan 已提交
427 428
}

S
Shengliang Guan 已提交
429 430 431
int32_t taosUnLockFile(int32_t fd) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
S
TD-4088  
Shengliang Guan 已提交
432
#else
S
Shengliang Guan 已提交
433 434 435
  return (int32_t)flock(fd, LOCK_UN | LOCK_NB);
#endif
}
S
TD-4088  
Shengliang Guan 已提交
436

S
Shengliang Guan 已提交
437 438 439 440 441 442 443
int32_t taosUmaskFile(int32_t val) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
  return umask(val);
#endif
}
S
TD-4088  
Shengliang Guan 已提交
444

S
Shengliang Guan 已提交
445 446 447 448 449 450
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
  struct stat fileStat;
  int32_t     code = stat(path, &fileStat);
S
TD-4088  
Shengliang Guan 已提交
451
  if (code < 0) {
S
Shengliang Guan 已提交
452
    return code;
S
TD-4088  
Shengliang Guan 已提交
453 454
  }

S
Shengliang Guan 已提交
455 456 457 458 459 460 461 462 463 464
  if (size != NULL) {
    *size = fileStat.st_size;
  }

  if (mtime != NULL) {
    *mtime = fileStat.st_mtime;
  }

  return 0;
#endif
S
TD-1912  
Shengliang Guan 已提交
465 466
}

S
Shengliang Guan 已提交
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
  struct stat fileStat;
  int32_t     code = fstat(fd, &fileStat);
  if (code < 0) {
    return code;
  }

  if (size != NULL) {
    *size = fileStat.st_size;
  }

  if (mtime != NULL) {
    *mtime = fileStat.st_mtime;
  }

  return 0;
#endif
}

int32_t taosOpenFileWrite(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
  return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}

497 498
int32_t taosOpenFileCreateWrite(const char *path) {
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
S
Shengliang Guan 已提交
499 500
  return 0;
#else
501
  return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO);
S
Shengliang Guan 已提交
502 503 504
#endif
}

505
int32_t taosOpenFileCreateWriteTrunc(const char *path) {
S
Shengliang Guan 已提交
506 507 508
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
509
  return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
S
Shengliang Guan 已提交
510 511 512
#endif
}

513
int32_t taosOpenFileCreateWriteAppend(const char *path) {
S
Shengliang Guan 已提交
514 515 516
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
517
  return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
S
TD-4088  
Shengliang Guan 已提交
518
#endif
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
}

FileFd taosOpenFileRead(const char *path) {
  #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
  return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}

FileFd taosOpenFileReadWrite(const char *path) {
  #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
  return 0;
#else
  return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
}