enhance: 1. fix api comment, 2. del dead code

mindspore/ccsrc/dataset/core/data_type.cc

@@ -18,7 +18,6 @@
 #include "utils/log_adapter.h"
 
 #include "dataset/core/pybind_support.h"
-#include "dataset/util/de_error.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/dataset/core/tensor.cc

@@ -152,7 +152,7 @@ Tensor::Tensor(const std::vector<std::string> &strings, const TensorShape &shape
 
   this->data_end_ = data_ + offset_arr[i];
 
-  DS_ASSERT(num_bytes == 0);
+  MS_ASSERT(num_bytes == 0);
   if (shape.known()) Tensor::Reshape(shape);
 }
 Tensor::Tensor(const dataengine::BytesList &bytes_list, const TensorShape &shape)
@@ -191,7 +191,7 @@ Tensor::Tensor(const dataengine::BytesList &bytes_list, const TensorShape &shape
 
   data_end_ = data_ + offset_arr[i];
 
-  DS_ASSERT(num_bytes == 0);
+  MS_ASSERT(num_bytes == 0);
   if (shape.known()) Tensor::Reshape(shape);
 }
 Status Tensor::CreateTensor(std::shared_ptr<Tensor> *ptr, TensorImpl tensor_impl, const TensorShape &shape,
@@ -420,7 +420,7 @@ bool Tensor::operator==(const Tensor &rhs) const {
 // Description: A function that print the value as specified by its index
 void Tensor::PrintItemAt(const std::vector<dsize_t> &index, std::ostream &out) const {
   Status rc;
-  DS_ASSERT(data_);
+  MS_ASSERT(data_);
 
   switch (type_.value()) {
     CASE_PRINT_HEX(DataType::DE_BOOL, bool);

mindspore/ccsrc/dataset/core/tensor.h

@@ -33,7 +33,6 @@
 #include "dataset/core/data_type.h"
 #include "dataset/core/tensor_shape.h"
 #include "dataset/util/allocator.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/status.h"
 #include "proto/example.pb.h"
 

mindspore/ccsrc/dataset/core/tensor_shape.cc

@@ -22,7 +22,6 @@
 #include "common/utils.h"
 #include "utils/log_adapter.h"
 #include "dataset/core/constants.h"
-#include "dataset/util/de_error.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/ccsrc/dataset/engine/connector.h

@@ -97,7 +97,7 @@ class Connector {
   virtual Status Pop(int32_t worker_id,  // The worker-id of the caller. See the requirement at the top of this file.
                      T *result) noexcept {
     {
-      DS_ASSERT(worker_id < num_consumers_);
+      MS_ASSERT(worker_id < num_consumers_);
       std::unique_lock<std::mutex> lk(m_);
       RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; }));
       RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result));
@@ -114,8 +114,8 @@ class Connector {
   // @param worker_id The id of a worker thread calling this method.
   // @param el A const lvalue element to be passed/added/pushed.
   Status Push(int32_t worker_id, const T &el) noexcept {
-    DS_ASSERT(worker_id < static_cast<int32_t>(queues_.size()));
-    DS_ASSERT(queues_[worker_id] != nullptr);
+    MS_ASSERT(worker_id < static_cast<int32_t>(queues_.size()));
+    MS_ASSERT(queues_[worker_id] != nullptr);
     return (queues_[worker_id]->Add(el));
   }
 
@@ -125,8 +125,8 @@ class Connector {
   // @param worker_id The id of a worker thread calling this method.
   // @param el An element to be passed/added/pushed.
   virtual Status Push(int32_t worker_id, T &&el) noexcept {
-    DS_ASSERT(worker_id < static_cast<int32_t>(queues_.size()));
-    DS_ASSERT(queues_[worker_id] != nullptr);
+    MS_ASSERT(worker_id < static_cast<int32_t>(queues_.size()));
+    MS_ASSERT(queues_[worker_id] != nullptr);
     return (queues_[worker_id]->Add(std::forward<T>(el)));
   }
 

mindspore/ccsrc/dataset/engine/data_schema.cc

@@ -27,7 +27,6 @@
 #include "dataset/util/status.h"
 #include "dataset/core/tensor_shape.h"
 #include "utils/log_adapter.h"
-#include "dataset/util/de_error.h"
 
 namespace mindspore {
 namespace dataset {
@@ -425,7 +424,7 @@ DataSchema::~DataSchema() = default;
 
 // Getter for the ColDescriptor by index
 const ColDescriptor &DataSchema::column(int32_t idx) const {
-  DS_ASSERT(idx < static_cast<int>(col_descs_.size()));
+  MS_ASSERT(idx < static_cast<int>(col_descs_.size()));
   return col_descs_[idx];
 }
 

mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc

@@ -110,7 +110,7 @@ void DatasetOp::RemoveParent(const DatasetOp *parent) {
 
 // Getter function to get a shared pointer to our childAdds a operator to become our child.
 std::shared_ptr<DatasetOp> DatasetOp::child(int32_t child_index) const {
-  DS_ASSERT(child_index < static_cast<int>(child_.size()));
+  MS_ASSERT(child_index < static_cast<int>(child_.size()));
   // Return a shared pointer
   return child_[child_index];
 }

mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc

@@ -104,7 +104,7 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) {
   // following loop puts the 2 level of shuffles together into 1 vector
   for (size_t ind = 0; ind < v.size(); ++ind) {
     while (v[ind]->second.empty() == false) {
-      DS_ASSERT(!(v[ind]->first.empty()));  // make sure that v[ind]->first.substr(1) is not out of bound
+      MS_ASSERT(!(v[ind]->first.empty()));  // make sure that v[ind]->first.substr(1) is not out of bound
       v[ind]->second.front()->second = class_index_.empty() ? ind : class_index_[v[ind]->first.substr(1)];
       image_label_pairs_.push_back(v[ind]->second.front());
       v[ind]->second.pop();

mindspore/ccsrc/dataset/engine/jagged_connector.h

@@ -44,7 +44,7 @@ class JaggedConnector : public Connector<std::unique_ptr<DataBuffer>> {
 
   Status Pop(int32_t worker_id, std::unique_ptr<DataBuffer> *result) noexcept override {
     {
-      DS_ASSERT(worker_id < num_consumers_);
+      MS_ASSERT(worker_id < num_consumers_);
       std::unique_lock<std::mutex> lock(m_);
       RETURN_IF_NOT_OK(cv_.Wait(&lock, [this, worker_id]() { return expect_consumer_ == worker_id; }));
       if (is_queue_finished_[pop_from_]) {

mindspore/ccsrc/dataset/util/arena.cc

@@ -17,7 +17,6 @@
 #include <unistd.h>
 #include <utility>
 #include "dataset/util/system_pool.h"
-#include "dataset/util/de_error.h"
 #include "./securec.h"
 #include "utils/log_adapter.h"
 
@@ -79,7 +78,7 @@ void Arena::Deallocate(void *p) {
   auto *q = get_base_addr(p);
   MemHdr hdr(0, 0);
   MemHdr::getHdr(q, &hdr);
-  DS_ASSERT(hdr.sig == 0xDEADBEEF);
+  MS_ASSERT(hdr.sig == 0xDEADBEEF);
   // We are going to insert a free block back to the treap. But first, check if we can combine
   // with the free blocks before and after to form a bigger block.
   std::unique_lock<std::mutex> lck(mux_);
@@ -103,8 +102,8 @@ void Arena::Deallocate(void *p) {
 }
 
 Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) {
-  DS_ASSERT(pp);
-  DS_ASSERT(*pp);
+  MS_ASSERT(pp);
+  MS_ASSERT(*pp);
   uint64_t actual_size = static_cast<uint64_t>(new_sz) + ARENA_WALL_OVERHEAD_SZ;
   if (actual_size > this->get_max_size()) {
     RETURN_STATUS_UNEXPECTED("Request size too big : " + std::to_string(new_sz));
@@ -114,7 +113,7 @@ Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) {
   auto *oldHdr = get_base_addr(oldAddr);
   MemHdr hdr(0, 0);
   MemHdr::getHdr(oldHdr, &hdr);
-  DS_ASSERT(hdr.sig == 0xDEADBEEF);
+  MS_ASSERT(hdr.sig == 0xDEADBEEF);
   std::unique_lock<std::mutex> lck(mux_);
   if (hdr.blk_size > req_blk) {
     // Refresh the header with the new smaller size.
@@ -239,8 +238,8 @@ bool Arena::BlockEnlarge(uint64_t *addr, uint64_t old_sz, uint64_t new_sz) {
 }
 
 Status Arena::FreeAndAlloc(void **pp, size_t old_sz, size_t new_sz) {
-  DS_ASSERT(pp);
-  DS_ASSERT(*pp);
+  MS_ASSERT(pp);
+  MS_ASSERT(*pp);
   void *p = nullptr;
   void *q = *pp;
   RETURN_IF_NOT_OK(Allocate(new_sz, &p));

mindspore/ccsrc/dataset/util/btree_impl.tpp

@@ -61,8 +61,8 @@ typename BPlusTree<K, V, A, C, T>::IndexRc BPlusTree<K, V, A, C, T>::InnerNode::
 template <typename K, typename V, typename A, typename C, typename T>
 typename BPlusTree<K, V, A, C, T>::IndexRc BPlusTree<K, V, A, C, T>::InnerNode::Split(
   BPlusTree<K, V, A, C, T>::InnerNode *to, key_type *split_key) {
-  DS_ASSERT(to);
-  DS_ASSERT(to->slotuse_ == 0);
+  MS_ASSERT(to);
+  MS_ASSERT(to->slotuse_ == 0);
   // It is simpler to sort first, then split. Other alternative is to move key by key to the
   // new node. Also we need to deal with the 'holes' after a key is moved.
   RETURN_IF_BAD_RC(this->Sort());
@@ -153,8 +153,8 @@ typename BPlusTree<K, V, A, C, T>::IndexRc BPlusTree<K, V, A, C, T>::LeafNode::S
 template <typename K, typename V, typename A, typename C, typename T>
 typename BPlusTree<K, V, A, C, T>::IndexRc BPlusTree<K, V, A, C, T>::LeafNode::Split(
   BPlusTree<K, V, A, C, T>::LeafNode *to) {
-  DS_ASSERT(to);
-  DS_ASSERT(to->slotuse_ == 0);
+  MS_ASSERT(to);
+  MS_ASSERT(to->slotuse_ == 0);
   // It is simpler to sort first, then split. Other alternative is to move key by key to the
   // new node. Also we need to deal with the 'holes' after a key is moved.
   RETURN_IF_BAD_RC(this->Sort());

mindspore/ccsrc/dataset/util/btree_iterator.tpp

@@ -15,7 +15,6 @@
 #ifndef DATASET_UTIL_BTREE_ITERATOR_H_
 #define DATASET_UTIL_BTREE_ITERATOR_H_
 
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 #include "btree.h"
 

mindspore/ccsrc/dataset/util/buddy.cc

@@ -16,9 +16,9 @@
 #include "dataset/util/buddy.h"
 #include <iomanip>
 #include <stdexcept>
-#include "dataset/util/de_error.h"
 #include "dataset/util/memory_pool.h"
 #include "dataset/util/system_pool.h"
+#include "utils/log_adapter.h"
 #include "./securec.h"
 
 inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); }
@@ -68,7 +68,7 @@ Status BuddySpace::Alloc(const uint64_t sz, BSpaceDescriptor *desc, addr_t *p) n
 }
 
 addr_t BuddySpace::AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexcept {
-  DS_ASSERT(sz <= max_);
+  MS_ASSERT(sz <= max_);
   uint32_t reqSize = SizeToBlock(sz);
   rel_addr_t rel_addr = AllocBuddySeg(reqSize);
   if (rel_addr != static_cast<rel_addr_t>(NOSPACE)) {
@@ -84,7 +84,7 @@ addr_t BuddySpace::AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexce
 }
 
 void BuddySpace::FreeNoLock(const BSpaceDescriptor *desc) {
-  DS_ASSERT(desc->sig == 0XDEADBEEF);
+  MS_ASSERT(desc->sig == 0XDEADBEEF);
   rel_addr_t rel_addr = desc->addr;
   size_t blk_size = desc->blk_size;
   size_t req_size = desc->req_size;
@@ -217,7 +217,7 @@ void BuddySpace::JoinBuddySeg(rel_addr_t addr, size_t blk_sz) {
       auto log_sz = static_cast<log_t>(Log2(blk_sz));
       rel_addr_t left = (buddy < addr) ? buddy : addr;
       rel_addr_t right = left + blk_sz;
-      DS_ASSERT(count_[log_sz] >= 2);
+      MS_ASSERT(count_[log_sz] >= 2);
       count_[log_sz] -= 2;
       SetBuddySegState(right, blk_sz, STATE::kEmpty);
       SetBuddySegState(left, BitLeftShift(blk_sz, 1), STATE::kFree);
@@ -235,7 +235,7 @@ void BuddySpace::JoinBuddySeg(rel_addr_t addr, size_t blk_sz) {
 }
 
 void BuddySpace::TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) {
-  DS_ASSERT(ask_sz < blk_sz);
+  MS_ASSERT(ask_sz < blk_sz);
   uint32_t inx = Log2(blk_sz);
   size_t remaining_sz = ask_sz;
   for (int i = inx; i > 0; i--) {
@@ -256,7 +256,7 @@ void BuddySpace::TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) {
 }
 
 void BuddySpace::UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) {
-  DS_ASSERT(ask_sz < blk_sz);
+  MS_ASSERT(ask_sz < blk_sz);
   uint32_t inx = Log2(blk_sz);
   size_t remaining_sz = ask_sz;
   for (int i = inx; i > 0; i--) {
@@ -268,7 +268,7 @@ void BuddySpace::UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) {
         size_t sz = 0;
         STATE st;
         GetBuddySegState(addr, &sz, &st);
-        DS_ASSERT(sz == half_sz && st == STATE::kAlloc);
+        MS_ASSERT(sz == half_sz && st == STATE::kAlloc);
       }
 #endif
       SetBuddySegState(addr, half_sz, STATE::kFree);
@@ -291,7 +291,7 @@ rel_addr_t BuddySpace::AllocBuddySeg(uint32_t req_size) noexcept {
   STATE st;
   size_t sz = 0;
   for (int i = start_inx; !found && i < num_lvl_; i++) {
-    DS_ASSERT(count_[i] >= 0);
+    MS_ASSERT(count_[i] >= 0);
     if (count_[i] == 0) {
       continue;
     }
@@ -302,7 +302,7 @@ rel_addr_t BuddySpace::AllocBuddySeg(uint32_t req_size) noexcept {
       if (st == STATE::kFree && sz == blk_sz) {
         found = true;
       } else {
-        DS_ASSERT(st != STATE::kEmpty);
+        MS_ASSERT(st != STATE::kEmpty);
         ask_addr += ((sz > blk_sz) ? sz : blk_sz);
       }
     }

mindspore/ccsrc/dataset/util/circular_pool.cc

@@ -19,8 +19,8 @@
 #include <limits>
 #include <utility>
 #include "./securec.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/system_pool.h"
+#include "utils/log_adapter.h"
 
 namespace mindspore {
 namespace dataset {
@@ -65,7 +65,7 @@ void CircularPool::CircularIterator::Reset() {
     auto list_end = dp_->mem_segments_.end();
     auto it = std::find_if(dp_->mem_segments_.begin(), list_end,
                            [this](const std::shared_ptr<Arena> &b) { return b.get() == cur_tail_; });
-    DS_ASSERT(it != list_end);
+    MS_ASSERT(it != list_end);
     start_ = std::distance(dp_->mem_segments_.begin(), it);
     cur_ = start_;
     has_next_ = true;
@@ -153,7 +153,7 @@ Status CircularPool::Reallocate(void **pp, size_t old_sz, size_t new_sz) {
     return (q > base && q < base + b->get_max_size());
   });
   lock.Unlock();
-  DS_ASSERT(it != mem_segments_.end());
+  MS_ASSERT(it != mem_segments_.end());
   Arena *ba = it->get();
   Status rc = ba->Reallocate(pp, old_sz, new_sz);
   if (rc.IsOutofMemory()) {

mindspore/ccsrc/dataset/util/de_error.h

-/**
- * Copyright 2019 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef DATASET_UTIL_DE_ERROR_H_
-#define DATASET_UTIL_DE_ERROR_H_
-
-#ifdef DEBUG
-#include <cassert>
-#define DS_ASSERT(f) assert(f)
-#else
-#define DS_ASSERT(f) ((void)0)
-#endif
-
-#include <map>
-#include "utils/error_code.h"
-
-namespace mindspore {
-namespace dataset {
-DE_ERRORNO_DATASET(CATCH_EXCEPTION, 0, "try catch exception error");
-DE_ERRORNO_DATASET(FILE_NOT_FOUND, 1, "file is not found");
-DE_ERRORNO_DATASET(PARSE_FAILED, 2, "parse failed");
-DE_ERRORNO_DATASET(COPY_ERROR, 3, "copy data error");
-DE_ERRORNO_DATASET(BOUND_ERROR, 4, "variable overflow or lost of precision");
-DE_ERRORNO_DATASET(ALLOC_FAILED, 5, "dynamic memory allocation failed");
-}  // namespace dataset
-}  // namespace mindspore
-#endif  // DATASET_UTIL_DE_ERROR_H_

mindspore/ccsrc/dataset/util/list.h

@@ -18,7 +18,8 @@
 
 #include <iostream>
 #include <iterator>
-#include "dataset/util/de_error.h"
+
+#include "utils/log_adapter.h"
 
 namespace mindspore {
 namespace dataset {
@@ -90,7 +91,7 @@ struct List {
 
   // Insert elem2 after elem1 in the list.
   virtual void InsertAfter(pointer elem1, pointer elem2) {
-    DS_ASSERT(elem1 != elem2);
+    MS_ASSERT(elem1 != elem2);
     Node<T> &elem1_node = elem1->*node;
     Node<T> &elem2_node = elem2->*node;
     elem2_node.prev = elem1;
@@ -108,7 +109,7 @@ struct List {
 
   // Insert elem2 before elem1 in the list.
   virtual void InsertBefore(pointer elem1, pointer elem2) {
-    DS_ASSERT(elem1 != elem2);
+    MS_ASSERT(elem1 != elem2);
     Node<T> &elem1_node = elem1->*node;
     Node<T> &elem2_node = elem2->*node;
     elem2_node.next = elem1;

mindspore/ccsrc/dataset/util/lock.cc

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 #include "dataset/util/lock.h"
-#include "dataset/util/de_error.h"
+#include "utils/log_adapter.h"
 
 namespace mindspore {
 namespace dataset {
@@ -63,7 +63,7 @@ void RWLock::Unlock() noexcept {
 
 void RWLock::Upgrade() {
   std::unique_lock<std::mutex> lck(mtx_);
-  DS_ASSERT(status_);
+  MS_ASSERT(status_);
   if (status_ == -1) {
     // I am a writer already.
     return;
@@ -81,7 +81,7 @@ void RWLock::Upgrade() {
 
 void RWLock::Downgrade() {
   std::unique_lock<std::mutex> lck(mtx_);
-  DS_ASSERT(status_);
+  MS_ASSERT(status_);
   if (status_ == -1) {
     // If there are no other writers waiting, just change the status
     if (waiting_writers_ == 0) {
@@ -111,24 +111,24 @@ SharedLock::~SharedLock() {
 }
 
 void SharedLock::Unlock() {
-  DS_ASSERT(ownlock_ == true);
+  MS_ASSERT(ownlock_ == true);
   rw_->Unlock();
   ownlock_ = false;
 }
 
 void SharedLock::Lock() {
-  DS_ASSERT(ownlock_ == false);
+  MS_ASSERT(ownlock_ == false);
   rw_->LockShared();
   ownlock_ = true;
 }
 
 void SharedLock::Upgrade() {
-  DS_ASSERT(ownlock_ == true);
+  MS_ASSERT(ownlock_ == true);
   rw_->Upgrade();
 }
 
 void SharedLock::Downgrade() {
-  DS_ASSERT(ownlock_ == true);
+  MS_ASSERT(ownlock_ == true);
   rw_->Downgrade();
 }
 
@@ -146,13 +146,13 @@ UniqueLock::~UniqueLock() {
 }
 
 void UniqueLock::Unlock() {
-  DS_ASSERT(ownlock_ == true);
+  MS_ASSERT(ownlock_ == true);
   rw_->Unlock();
   ownlock_ = false;
 }
 
 void UniqueLock::Lock() {
-  DS_ASSERT(ownlock_ == false);
+  MS_ASSERT(ownlock_ == false);
   rw_->LockExclusive();
   ownlock_ = true;
 }
@@ -171,13 +171,13 @@ LockGuard::~LockGuard() {
 }
 
 void LockGuard::Unlock() {
-  DS_ASSERT(own_lock_);
+  MS_ASSERT(own_lock_);
   lck_->Unlock();
   own_lock_ = false;
 }
 
 void LockGuard::Lock() {
-  DS_ASSERT(own_lock_ == false);
+  MS_ASSERT(own_lock_ == false);
   lck_->Lock();
   own_lock_ = true;
 }

mindspore/ccsrc/dataset/util/path.cc

@@ -23,7 +23,6 @@
 #include <utility>
 
 #include "common/utils.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 namespace mindspore {

mindspore/ccsrc/dataset/util/queue.h

@@ -25,7 +25,6 @@
 #include <vector>
 
 #include "common/utils.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 #include "dataset/util/allocator.h"
 #include "dataset/util/services.h"

mindspore/ccsrc/dataset/util/storage_container.cc

@@ -20,7 +20,6 @@
 #include <unistd.h>
 #include <vector>
 #include "common/utils.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/path.h"
 #include "dataset/util/status.h"
 #include "utils/log_adapter.h"
@@ -59,7 +58,7 @@ Status StorageContainer::Close() noexcept {
 }
 
 Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcept {
-  DS_ASSERT(is_open_);
+  MS_ASSERT(is_open_);
   RETURN_UNEXPECTED_IF_NULL(dest);
   auto sz = dest->GetSize();
 #if defined(_WIN32) || defined(_WIN64)
@@ -83,7 +82,7 @@ Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcep
 }
 
 Status StorageContainer::Write(const ReadableSlice &dest, off64_t offset) const noexcept {
-  DS_ASSERT(is_open_);
+  MS_ASSERT(is_open_);
   auto sz = dest.GetSize();
 #if defined(_WIN32) || defined(_WIN64)
   // Doesn't seem there is any pwrite64 on mingw.

mindspore/ccsrc/dataset/util/storage_manager.cc

@@ -22,7 +22,6 @@
 #include "common/utils.h"
 #include "dataset/util/path.h"
 #include "dataset/util/services.h"
-#include "dataset/util//de_error.h"
 #include "utils/log_adapter.h"
 
 namespace mindspore {

mindspore/ccsrc/dataset/util/task.cc

@@ -16,7 +16,6 @@
 #include "dataset/util/task.h"
 #include "common/utils.h"
 #include "dataset/util/task_manager.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 namespace mindspore {

mindspore/ccsrc/dataset/util/task.h

@@ -27,7 +27,6 @@
 #include <stdexcept>
 #include <string>
 #include <thread>
-#include "dataset/util/de_error.h"
 #include "dataset/util/intrp_resource.h"
 #include "dataset/util/list.h"
 #include "dataset/util/memory_pool.h"

mindspore/ccsrc/utils/error_code.h

-/**
- * Copyright 2019 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_
-#define MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_
-
-#include <map>
-#include <string>
-
-namespace DataEngineBase {
-// system ID
-const int SYSID_MD = 20;
-
-// Runtime location
-enum LogRuntime {
-  RT_HOST = 0b01,
-  RT_DEVICE = 0b10,
-};
-
-// sub model
-enum SubModuleId {
-  COMMON_MODULE = 0,
-  DATAGET_MODULE,
-  MINDRECORD_MODULE,
-};
-
-// error code type
-enum ErrorCodeType {
-  ERROR_CODE = 0b01,
-  EXCEPTION_CODE = 0b10,
-};
-
-// error level
-enum ErrorLevel {
-  COMMON_LEVEL = 0b000,
-  SUGGESTION_LEVEL = 0b001,
-  MINOR_LEVEL = 0b010,
-  MAJOR_LEVEL = 0b011,
-  CRITICAL_LEVEL = 0b100,
-};
-
-// code compose(4 byte), runtime: 2 bit,  type: 2 bit,   level: 3 bit,  sysid: 8 bit, modid: 5 bit, value: 12 bit
-#define DE_ERRORNO(runtime, type, level, sysid, submodid, name, value, desc)                                       \
-  constexpr DataEngineBase::Status name = ((0xFF & ((uint8_t)runtime)) << 30) | ((0xFF & ((uint8_t)type)) << 28) | \
-                                          ((0xFF & ((uint8_t)level)) << 25) | ((0xFF & ((uint8_t)sysid)) << 17) |  \
-                                          ((0xFF & ((uint8_t)submodid)) << 12) | (0x0FFF & ((uint16_t)value));     \
-  const DataEngineBase::ErrorNoRegisterar g_##name##_errorno(name, desc);
-
-// each module defines error codes using the following macros
-#define DE_ERRORNO_COMMON(name, value, desc)                                                    \
-  DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \
-             DataEngineBase::SYSID_MD, DataEngineBase::COMMON_MODULE, name, value, desc)
-
-#define DE_ERRORNO_DATASET(name, value, desc)                                                   \
-  DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \
-             DataEngineBase::SYSID_MD, DataEngineBase::DATAGET_MODULE, name, value, desc)
-
-#define DE_ERRORNO_MINDRECORD(name, value, desc)                                                \
-  DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \
-             DataEngineBase::SYSID_MD, DataEngineBase::MINDRECORD_MODULE, name, value, desc)
-
-// get error code description
-#define DE_GET_ERRORNO_STR(value) DataEngineBase::StatusFactory::Instance()->GetErrDesc(value)
-
-class StatusFactory {
- public:
-  static StatusFactory *Instance() {
-    static StatusFactory instance;
-    return &instance;
-  }
-
-  void RegisterErrorNo(uint32_t err, const std::string &desc) {
-    if (err_desc_.find(err) != err_desc_.end()) return;
-    err_desc_[err] = desc;
-  }
-
-  std::string GetErrDesc(uint32_t err) {
-    auto iter_find = err_desc_.find(err);
-    if (iter_find == err_desc_.end()) return "";
-    return iter_find->second;
-  }
-
- protected:
-  StatusFactory() = default;
-
-  ~StatusFactory() = default;
-
- private:
-  std::map<uint32_t, std::string> err_desc_;
-};
-
-class ErrorNoRegisterar {
- public:
-  ErrorNoRegisterar(uint32_t err, const std::string &desc) { StatusFactory::Instance()->RegisterErrorNo(err, desc); }
-
-  ~ErrorNoRegisterar() = default;
-};
-
-using Status = uint32_t;
-}  // namespace DataEngineBase
-#endif  // MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_

mindspore/mindrecord/filewriter.py

@@ -240,7 +240,9 @@ class FileWriter:
 
     def set_header_size(self, header_size):
         """
-        Set the size of header.
+        Set the size of header which contains shard information, schema information, \
+        page meta information, etc. The larger the header, the more training data \
+        a single mindrecord file can store.
 
         Args:
             header_size (int): Size of header, between 16KB and 128MB.
@@ -256,7 +258,9 @@ class FileWriter:
 
     def set_page_size(self, page_size):
         """
-        Set the size of Page.
+        Set the size of page which mainly refers to the block to store training data, \
+        and the training data will be split into raw page and blob page in mindrecord. \
+        The larger the page, the more training data a single page can store.
 
         Args:
            page_size (int): Size of page, between 32KB and 256MB.

tests/ut/cpp/dataset/arena_test.cc

@@ -17,7 +17,6 @@
 #include <string>
 #include "dataset/util/arena.h"
 #include "common/common.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/batch_op_test.cc

@@ -21,7 +21,6 @@
 #include "common/utils.h"
 #include "gtest/gtest.h"
 #include "dataset/core/global_context.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 #include "securec.h"
 #include "dataset/util/status.h"

tests/ut/cpp/dataset/btree_test.cc

@@ -21,7 +21,6 @@
 #include "dataset/util/task_manager.h"
 #include "common/common.h"
 #include "gtest/gtest.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/celeba_op_test.cc

@@ -23,7 +23,6 @@
 #include "dataset/core/global_context.h"
 #include "dataset/engine/datasetops/source/celeba_op.h"
 #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/status.h"
 #include "gtest/gtest.h"
 #include "utils/log_adapter.h"

tests/ut/cpp/dataset/cifar_op_test.cc

@@ -26,7 +26,6 @@
 #include "dataset/engine/datasetops/source/sampler/sampler.h"
 #include "dataset/engine/datasetops/source/sampler/random_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/path.h"
 #include "dataset/util/status.h"
 #include "gtest/gtest.h"

tests/ut/cpp/dataset/coco_op_test.cc

@@ -30,7 +30,6 @@
 #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/path.h"
 #include "dataset/util/status.h"
 #include "gtest/gtest.h"

tests/ut/cpp/dataset/common/common.h

@@ -16,7 +16,6 @@
 #ifndef TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_
 #define TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_
 
-#include "dataset/util/de_error.h"
 #include "gtest/gtest.h"
 #include "utils/log_adapter.h"
 

tests/ut/cpp/dataset/connector_test.cc

@@ -140,7 +140,7 @@ Status MindDataTestConnector::Run_test_0() {
   auto my_conn = std::make_shared<Connector<uint32_t>>(1,  // num of producers
                                                       1,  // num of consumers
                                                       10);  // capacity of each queue
-  DS_ASSERT(my_conn != nullptr);
+  MS_ASSERT(my_conn != nullptr);
 
   rc = my_conn->Register(tg_.get());
   RETURN_IF_NOT_OK(rc);
@@ -277,7 +277,7 @@ Status MindDataTestConnector::FirstWorkerPush(
                                       int start_in,
                                       int offset)  {
   TaskManager::FindMe()->Post();
-  DS_ASSERT(my_conn != nullptr);
+  MS_ASSERT(my_conn != nullptr);
   Status rc;
   for (int i = start_in; i < input_.size(); i += offset) {
     rc = my_conn->Push(tid, input_[i]);
@@ -294,7 +294,7 @@ Status MindDataTestConnector::MidWorkerJob(
                                    int tid,
                                    std::shared_ptr<Connector<uint32_t> > from_conn,
                                    std::shared_ptr<Connector<uint32_t> > to_conn) {
-  DS_ASSERT((from_conn != nullptr) && (to_conn != nullptr));
+  MS_ASSERT((from_conn != nullptr) && (to_conn != nullptr));
   Status rc;
   TaskManager::FindMe()->Post();
   while (1) {

tests/ut/cpp/dataset/duplicate_op_test.cc

@@ -17,7 +17,6 @@
 #include "common/common.h"
 #include "gtest/gtest.h"
 #include "dataset/core/tensor.h"
-#include "dataset/util/de_error.h"
 #include "dataset/kernels/data/duplicate_op.h"
 
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/execution_tree_test.cc

@@ -21,7 +21,6 @@
 #include "dataset/engine/datasetops/source/tf_reader_op.h"
 #include "common/common.h"
 #include "gtest/gtest.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/global_context_test.cc

@@ -15,7 +15,6 @@
  */
 #include "dataset/core/global_context.h"
 #include "common/common.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/image_folder_op_test.cc

@@ -29,7 +29,6 @@
 #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/path.h"
 #include "dataset/util/status.h"
 #include "gtest/gtest.h"

tests/ut/cpp/dataset/interrupt_test.cc

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 #include "common/common.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 #include "dataset/util/services.h"
 #include "dataset/util/intrp_service.h"

tests/ut/cpp/dataset/manifest_op_test.cc

@@ -25,7 +25,6 @@
 #include "dataset/engine/datasetops/source/manifest_op.h"
 #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/status.h"
 #include "gtest/gtest.h"
 #include "utils/log_adapter.h"

tests/ut/cpp/dataset/mask_test.cc

@@ -22,7 +22,6 @@
 #include "dataset/core/tensor.h"
 #include "dataset/core/cv_tensor.h"
 #include "dataset/core/data_type.h"
-#include "dataset/util/de_error.h"
 #include "dataset/kernels/data/mask_op.h"
 #include "dataset/kernels/data/data_utils.h"
 

tests/ut/cpp/dataset/mnist_op_test.cc

@@ -30,7 +30,6 @@
 #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/path.h"
 #include "dataset/util/status.h"
 #include "gtest/gtest.h"

tests/ut/cpp/dataset/path_test.cc

@@ -16,7 +16,6 @@
 #include "dataset/util/path.h"
 #include "common/common.h"
 #include "gtest/gtest.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 #include <cstdio>
 

tests/ut/cpp/dataset/stand_alone_samplers_test.cc

@@ -21,7 +21,6 @@
 #include "dataset/engine/datasetops/source/sampler/random_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/sampler.h"
 #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/status.h"
 #include "gtest/gtest.h"
 #include "utils/log_adapter.h"

tests/ut/cpp/dataset/status_test.cc

@@ -16,7 +16,6 @@
 #include "dataset/util/status.h"
 #include "common/common.h"
 #include "gtest/gtest.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/tensor_string_test.cc

@@ -22,7 +22,6 @@
 #include "dataset/core/tensor.h"
 #include "dataset/core/cv_tensor.h"
 #include "dataset/core/data_type.h"
-#include "dataset/util/de_error.h"
 
 using namespace mindspore::dataset;
 

tests/ut/cpp/dataset/tensor_test.cc

@@ -22,7 +22,6 @@
 #include "dataset/core/tensor.h"
 #include "dataset/core/cv_tensor.h"
 #include "dataset/core/data_type.h"
-#include "dataset/util/de_error.h"
 
 using namespace mindspore::dataset;
 

tests/ut/cpp/dataset/treap_test.cc

@@ -16,7 +16,6 @@
 #include "dataset/util/treap.h"
 #include "common/common.h"
 #include "gtest/gtest.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/type_cast_op_test.cc

@@ -25,7 +25,6 @@
 #include "dataset/core/pybind_support.h"
 #include "gtest/gtest.h"
 #include "securec.h"
-#include "dataset/util/de_error.h"
 
 #define  MAX_INT_PRECISION 16777216  // float int precision is 16777216
 using namespace mindspore::dataset;

tests/ut/cpp/dataset/voc_op_test.cc

@@ -30,7 +30,6 @@
 #include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
 #include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
-#include "dataset/util/de_error.h"
 #include "dataset/util/path.h"
 #include "dataset/util/status.h"
 #include "gtest/gtest.h"

tests/ut/cpp/dataset/zip_op_test.cc

@@ -32,7 +32,6 @@
 #include "dataset/engine/data_buffer.h"
 #include "gtest/gtest.h"
 #include "dataset/core/global_context.h"
-#include "dataset/util/de_error.h"
 #include "utils/log_adapter.h"
 
 namespace common = mindspore::common;