提交 2269c0d9 编写于 作者: X Xu Peng 提交者: xj.lin

feat(db): mock build_index related


Former-commit-id: eea24e1c5846af870694a10e865140efb0566a42
上级 817d032a
...@@ -166,14 +166,14 @@ Status DBImpl::background_merge_files(const std::string& group_id) { ...@@ -166,14 +166,14 @@ Status DBImpl::background_merge_files(const std::string& group_id) {
Status DBImpl::build_index(const meta::GroupFileSchema& file) { Status DBImpl::build_index(const meta::GroupFileSchema& file) {
//PXU TODO //PXU TODO
std::cout << ">>Building Index for: " << file.location << std::endl;
return Status::OK(); return Status::OK();
} }
Status DBImpl::background_build_index() { Status DBImpl::background_build_index() {
assert(bg_build_index_started_); assert(bg_build_index_started_);
meta::GroupFilesSchema to_index_files; meta::GroupFilesSchema to_index_files;
// PXU TODO _pMeta->files_to_index(to_index_files);
/* _pMeta->files_to_index(to_index_files); */
Status status; Status status;
for (auto& file : to_index_files) { for (auto& file : to_index_files) {
status = build_index(file); status = build_index(file);
......
#include <sys/stat.h>
#include <unistd.h>
#include <sstream> #include <sstream>
#include <iostream> #include <iostream>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <fstream>
#include "DBMetaImpl.h" #include "DBMetaImpl.h"
#include "IDGenerator.h" #include "IDGenerator.h"
...@@ -9,6 +12,13 @@ namespace vecwise { ...@@ -9,6 +12,13 @@ namespace vecwise {
namespace engine { namespace engine {
namespace meta { namespace meta {
long GetFileSize(const std::string& filename)
{
struct stat stat_buf;
int rc = stat(filename.c_str(), &stat_buf);
return rc == 0 ? stat_buf.st_size : -1;
}
DBMetaImpl::DBMetaImpl(const MetaOptions& options_) DBMetaImpl::DBMetaImpl(const MetaOptions& options_)
: _options(static_cast<const DBMetaOptions&>(options_)) { : _options(static_cast<const DBMetaOptions&>(options_)) {
initialize(); initialize();
...@@ -57,6 +67,24 @@ Status DBMetaImpl::add_group_file(const std::string& group_id, ...@@ -57,6 +67,24 @@ Status DBMetaImpl::add_group_file(const std::string& group_id,
return Status::OK(); return Status::OK();
} }
Status DBMetaImpl::files_to_index(GroupFilesSchema& files) {
// PXU TODO
files.clear();
std::stringstream ss;
ss << "/tmp/test/" << Meta::GetDate();
boost::filesystem::path path(ss.str().c_str());
boost::filesystem::directory_iterator end_itr;
for (boost::filesystem::directory_iterator itr(path); itr != end_itr; ++itr) {
std::cout << itr->path().string() << std::endl;
GroupFileSchema f;
f.location = itr->path().string();
if (1024*1024*50 >= GetFileSize(f.location)) continue;
std::cout << "About to index " << f.location << std::endl;
files.push_back(f);
}
return Status::OK();
}
Status DBMetaImpl::files_to_merge(const std::string& group_id, Status DBMetaImpl::files_to_merge(const std::string& group_id,
DatePartionedGroupFilesSchema& files) { DatePartionedGroupFilesSchema& files) {
//PXU TODO //PXU TODO
...@@ -72,6 +100,8 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id, ...@@ -72,6 +100,8 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id,
std::cout << itr->path().string() << std::endl; std::cout << itr->path().string() << std::endl;
GroupFileSchema f; GroupFileSchema f;
f.location = itr->path().string(); f.location = itr->path().string();
if (1024*1024*50 < GetFileSize(f.location)) continue;
std::cout << "About to merge " << f.location << std::endl;
files[date].push_back(f); files[date].push_back(f);
} }
......
...@@ -41,6 +41,8 @@ public: ...@@ -41,6 +41,8 @@ public:
virtual Status files_to_merge(const std::string& group_id, virtual Status files_to_merge(const std::string& group_id,
DatePartionedGroupFilesSchema& files) override; DatePartionedGroupFilesSchema& files) override;
virtual Status files_to_index(GroupFilesSchema&) override;
private: private:
Status initialize(); Status initialize();
......
...@@ -77,6 +77,8 @@ public: ...@@ -77,6 +77,8 @@ public:
virtual Status files_to_merge(const std::string& group_id, virtual Status files_to_merge(const std::string& group_id,
DatePartionedGroupFilesSchema& files) = 0; DatePartionedGroupFilesSchema& files) = 0;
virtual Status files_to_index(GroupFilesSchema&) = 0;
static DateT GetDate(const std::time_t& t); static DateT GetDate(const std::time_t& t);
static DateT GetDate(); static DateT GetDate();
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册