diff --git a/cube/cube-builder/include/cube-builder/crovl_builder_increment.h b/cube/cube-builder/include/cube-builder/crovl_builder_increment.h index d68ddc8010c27432ee03af0bc3ca69b97450ba1b..fa416bc4819e3fb6ac67e4676b8b1f56420f277f 100644 --- a/cube/cube-builder/include/cube-builder/crovl_builder_increment.h +++ b/cube/cube-builder/include/cube-builder/crovl_builder_increment.h @@ -27,7 +27,6 @@ class CROVLBuilderIncremental { static const uint32_t INDEX_BUFFER = 4194304; int _index_type; uint32_t _data_file_len; - // hdfs _index_file; uint64_t _data_file; uint32_t _data_buf_len; uint32_t _index_buf_len; @@ -39,12 +38,6 @@ class CROVLBuilderIncremental { uint32_t _fix_len; uint32_t _rec_size; - // hdfsFS _fs; - // hdfsFile _file; - - // AfsFileSystem* _afs_conn; - // Writer* _index_writer; - uint64_t* _index; char* _data; std::vector _data_file_list; @@ -74,15 +67,12 @@ class CROVLBuilderIncremental { std::string _id; std::string _key; std::string _extra; - // std::string _afs_regin; public: CROVLBuilderIncremental(); ~CROVLBuilderIncremental(); - // bool set_sort_arg(uint64_t inner_sort_size, uint64_t memory_quota); - bool Init(int index_type, uint32_t data_file_len, const char* mode, diff --git a/cube/cube-builder/include/cube-builder/raw_reader.h b/cube/cube-builder/include/cube-builder/raw_reader.h index 911478f649ddc0075ff53c0e58eedec0d10412ad..2aec4aa54942e49fa319137e1169562265635d8a 100644 --- a/cube/cube-builder/include/cube-builder/raw_reader.h +++ b/cube/cube-builder/include/cube-builder/raw_reader.h @@ -82,7 +82,7 @@ class FileRawReader : public RawReader { return -1; } virtual int64_t read(char* v) { - _reader.read(reinterpret_cast(*v), sizeof(char)); + _reader.read(reinterpret_cast(v), sizeof(char)); if (_reader.good()) return sizeof(char); else if (_reader.eof()) diff --git a/cube/cube-builder/include/cube-builder/seqfile_reader.h b/cube/cube-builder/include/cube-builder/seqfile_reader.h index 07a7fdf54b86cf2edaed52df7c1e44b286632487..c4e2b4a0a6013e5978b57b066b0d86a141ca1ba3 100644 --- a/cube/cube-builder/include/cube-builder/seqfile_reader.h +++ b/cube/cube-builder/include/cube-builder/seqfile_reader.h @@ -22,11 +22,6 @@ class SequenceFileRecordReader : public RecordReader { public: SequenceFileRecordReader() {} - // SequenceFileRecordReader(const std::string& path, AfsFileSystemPtr fs) { - // _path = path; - // _fs = fs; - // _raw_reader = new AFsRawReader(_path, _fs); - // } explicit SequenceFileRecordReader(const std::string& path) { _path = path; _raw_reader = new FileRawReader(_path); @@ -44,7 +39,6 @@ class SequenceFileRecordReader : public RecordReader { private: std::string _path{""}; - // AfsFileSystemPtr _fs {nullptr}; RawReader* _raw_reader{nullptr}; Header _header; }; diff --git a/cube/cube-builder/include/cube-builder/vtext.h b/cube/cube-builder/include/cube-builder/vtext.h index 8b03529027c81a43247fbb919cd4cd5f64c353e0..f7e31ac4a85bed271e9630f153b404db27f3e9e7 100644 --- a/cube/cube-builder/include/cube-builder/vtext.h +++ b/cube/cube-builder/include/cube-builder/vtext.h @@ -32,33 +32,6 @@ class VInt { static bool is_negative_vint(const char &value) { return value < -120 || (value >= -112 && value < 0); } - // todo - /*static int32_t writeVInt(OutputStream& os, int i) { - int32_t rc = 0; - if (i >= -112 && i <= 127) { - rc = os.write((char*)&i, (int64_t)sizeof(char)); - return rc; - } - int len = -112; - if (i < 0) { - i ^= -1L; // take one's complement' - len = -120; - } - int32_t tmp = i; - while (tmp != 0) { - tmp = tmp >> 8; - len--; - } - os.write((char*)&len, sizeof(char)); - len = (len < -120) ? -(len + 120) : -(len + 112); - for (int idx = len - 1; idx >= 0; idx--) { - int32_t shiftbits = idx * 8; - int mask = 0xFFL << shiftbits; - int val = (i & mask) >> shiftbits; - os.write((char*)&val, sizeof(char)); - } - return rc; - }*/ static bool read_vint(RawReader *reader, int32_t *vint) { char first_byte; if (reader->read(&first_byte) <= 0) { diff --git a/cube/cube-builder/src/builder_job.cpp b/cube/cube-builder/src/builder_job.cpp index cfaf1576bcbf3a3120f185d28fabf2cc7c370c8a..dfae8737c89e864c5a821dcb915ce61fb9768f98 100644 --- a/cube/cube-builder/src/builder_job.cpp +++ b/cube/cube-builder/src/builder_job.cpp @@ -61,11 +61,6 @@ void mapFileLocal(Job job, uint64_t key = *reinterpret_cast(const_cast(record.key.data())); - // printf("key:%lu yu:%lu value:%s \n", key, key%job.get_shard_num(), - // string_to_hex(record.value).c_str()); - // printf("key_length:%llu value_length:%llu \n", record.key.length(), - // record.value.length()); - // fprintf(stdout, "key:%lu value:%s\n", key, record.value.c_str()); total_count++; int part = key % job.get_shard_num(); int64_t value_length = record.record_len - record.key_len; diff --git a/cube/cube-builder/src/crovl_builder_increment.cpp b/cube/cube-builder/src/crovl_builder_increment.cpp index 512047b7cb5d61f5a015ebd077ce203e00ff28e5..9c90404ac8cda805bd4a83959d10e87328188641 100644 --- a/cube/cube-builder/src/crovl_builder_increment.cpp +++ b/cube/cube-builder/src/crovl_builder_increment.cpp @@ -156,10 +156,6 @@ bool CROVLBuilderIncremental::Init(int index_type, if (_master_address == "") { if (!read_last_meta_from_local()) { LOG(ERROR) << "read last meta from db error! "; - // if (!read_last_meta_from_hdfs()) { - // fprintf(stderr, "read last meta from hdfs error\n"); - // return false; - //} return false; } } else { @@ -182,7 +178,7 @@ bool CROVLBuilderIncremental::flush_data() { } char file[MAX_DATA_DIR_LEN * 2]; - snprintf(file, sizeof(file), "%s/data.%llu", _data_dir, _data_file); + snprintf(file, sizeof(file), "%s/data.%lu", _data_dir, _data_file); FILE *fp; if ((fp = fopen(file, "wb")) == NULL) { LOG(ERROR) << "open file failed! " << file; @@ -390,7 +386,7 @@ bool CROVLBuilderIncremental::done() { // fix empty data bug // if the version no data, filter this version, index_file_num no add - // only can patch is null, not bae + // only can patch is null, not base if (_cur_count > 0) { *reinterpret_cast(buffer) = ++_index_file_num; } else { diff --git a/cube/cube-builder/src/curl_simple.cpp b/cube/cube-builder/src/curl_simple.cpp index b8fb13442f4cc69361df6c4b64df689efb6c61f5..37d99116e34018d290b186279b4d47c2ce38c7d2 100644 --- a/cube/cube-builder/src/curl_simple.cpp +++ b/cube/cube-builder/src/curl_simple.cpp @@ -143,9 +143,6 @@ std::string CurlSimple::curl_post( if (res != CURLE_OK) { std::cerr << "curl_easy_perform() failed:" << curl_easy_strerror(res) << std::endl; - // return ""; - // fprintf(stderr, "curl_easy_perform() failed: %s\n", - // curl_easy_strerror(res)); } else { /* * Now, our chunk.memory points to a memory block that is chunk.size diff --git a/cube/cube-builder/src/main.cpp b/cube/cube-builder/src/main.cpp index 119db93e7db5e184c41af889af98ef1c7a867520..b9ba453b6ee39ad43958bbdfe68913b4516eac0b 100644 --- a/cube/cube-builder/src/main.cpp +++ b/cube/cube-builder/src/main.cpp @@ -44,8 +44,6 @@ int main(int argc, char *argv[]) { LOG(INFO) << "start"; - // string f = "/"; - // string output_path = argv[7] + f + depend_version + "_" + cur_version; string last_version = std::to_string(FLAGS_last_version); string cur_version = std::to_string(FLAGS_cur_version); string depend_version = std::to_string(FLAGS_depend_version);