diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 62821c556cb9c29403cd3fb321d95a69b02a26dc..026f34b64b3b3e43f8ba6519d3ce5f5b4b6d5e26 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -166,14 +166,14 @@ Status DBImpl::background_merge_files(const std::string& group_id) { Status DBImpl::build_index(const meta::GroupFileSchema& file) { //PXU TODO + std::cout << ">>Building Index for: " << file.location << std::endl; return Status::OK(); } Status DBImpl::background_build_index() { assert(bg_build_index_started_); meta::GroupFilesSchema to_index_files; - // PXU TODO - /* _pMeta->files_to_index(to_index_files); */ + _pMeta->files_to_index(to_index_files); Status status; for (auto& file : to_index_files) { status = build_index(file); diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index a9f7b1bf862c01cec391e81315a9ccd806c4ebd8..fd37038bfcf652183f5e0d3165b95e37b34b661c 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -1,6 +1,9 @@ +#include +#include #include #include #include +#include #include "DBMetaImpl.h" #include "IDGenerator.h" @@ -9,6 +12,13 @@ namespace vecwise { namespace engine { namespace meta { +long GetFileSize(const std::string& filename) +{ + struct stat stat_buf; + int rc = stat(filename.c_str(), &stat_buf); + return rc == 0 ? stat_buf.st_size : -1; +} + DBMetaImpl::DBMetaImpl(const MetaOptions& options_) : _options(static_cast(options_)) { initialize(); @@ -57,6 +67,24 @@ Status DBMetaImpl::add_group_file(const std::string& group_id, return Status::OK(); } +Status DBMetaImpl::files_to_index(GroupFilesSchema& files) { + // PXU TODO + files.clear(); + std::stringstream ss; + ss << "/tmp/test/" << Meta::GetDate(); + boost::filesystem::path path(ss.str().c_str()); + boost::filesystem::directory_iterator end_itr; + for (boost::filesystem::directory_iterator itr(path); itr != end_itr; ++itr) { + std::cout << itr->path().string() << std::endl; + GroupFileSchema f; + f.location = itr->path().string(); + if (1024*1024*50 >= GetFileSize(f.location)) continue; + std::cout << "About to index " << f.location << std::endl; + files.push_back(f); + } + return Status::OK(); +} + Status DBMetaImpl::files_to_merge(const std::string& group_id, DatePartionedGroupFilesSchema& files) { //PXU TODO @@ -72,6 +100,8 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, std::cout << itr->path().string() << std::endl; GroupFileSchema f; f.location = itr->path().string(); + if (1024*1024*50 < GetFileSize(f.location)) continue; + std::cout << "About to merge " << f.location << std::endl; files[date].push_back(f); } diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h index 4bf159cfd4b29c329396534297c20ead25df2a6c..ade7ced50a3a73a9851fc37d5954fac808f4e8e0 100644 --- a/cpp/src/db/DBMetaImpl.h +++ b/cpp/src/db/DBMetaImpl.h @@ -41,6 +41,8 @@ public: virtual Status files_to_merge(const std::string& group_id, DatePartionedGroupFilesSchema& files) override; + virtual Status files_to_index(GroupFilesSchema&) override; + private: Status initialize(); diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h index 73f9ebeddf9c8dd7c5e66991addf82927c63772b..847b31bef49f1dca08bbffc0bdb335ff82ebc93d 100644 --- a/cpp/src/db/Meta.h +++ b/cpp/src/db/Meta.h @@ -77,6 +77,8 @@ public: virtual Status files_to_merge(const std::string& group_id, DatePartionedGroupFilesSchema& files) = 0; + virtual Status files_to_index(GroupFilesSchema&) = 0; + static DateT GetDate(const std::time_t& t); static DateT GetDate();