提交 3beb766d 编写于 作者: 李寅

Merge branch 'optimize' into 'master'

Optimize load model data

See merge request !981
...@@ -280,11 +280,13 @@ int Main(int argc, char **argv) { ...@@ -280,11 +280,13 @@ int Main(int argc, char **argv) {
} }
} }
std::vector<unsigned char> model_weights_data; const unsigned char *model_weights_data = nullptr;
size_t model_weights_data_size = 0;
if (FLAGS_model_data_file != "") { if (FLAGS_model_data_file != "") {
if (!mace::ReadBinaryFile(&model_weights_data, FLAGS_model_data_file)) { MemoryMap(FLAGS_model_data_file,
LOG(FATAL) << "Failed to read file: " << FLAGS_model_data_file; &model_weights_data,
} &model_weights_data_size);
MACE_CHECK(model_weights_data != nullptr && model_weights_data_size != 0);
} }
#ifdef MODEL_GRAPH_FORMAT_CODE #ifdef MODEL_GRAPH_FORMAT_CODE
...@@ -300,8 +302,8 @@ int Main(int argc, char **argv) { ...@@ -300,8 +302,8 @@ int Main(int argc, char **argv) {
create_engine_status = create_engine_status =
CreateMaceEngineFromProto(model_graph_data.data(), CreateMaceEngineFromProto(model_graph_data.data(),
model_graph_data.size(), model_graph_data.size(),
model_weights_data.data(), model_weights_data,
model_weights_data.size(), model_weights_data_size,
input_names, input_names,
output_names, output_names,
config, config,
...@@ -378,6 +380,10 @@ int Main(int argc, char **argv) { ...@@ -378,6 +380,10 @@ int Main(int argc, char **argv) {
statistician->PrintStat(); statistician->PrintStat();
if (model_weights_data != nullptr) {
MemoryUnMap(model_weights_data, model_weights_data_size);
}
return 0; return 0;
} }
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
#include <algorithm> #include <algorithm>
#include <cerrno> #include <cerrno>
#include <cstring> #include <cstring>
#include <fstream>
#include <string> #include <string>
#include <utility> #include <utility>
#include <vector> #include <vector>
......
...@@ -13,9 +13,14 @@ ...@@ -13,9 +13,14 @@
// limitations under the License. // limitations under the License.
#include <dirent.h> #include <dirent.h>
#include <fcntl.h>
#include <malloc.h> #include <malloc.h>
#include <stdint.h> #include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <cstdint>
#include <cstdlib> #include <cstdlib>
#include <cstring>
#include <fstream> #include <fstream>
#include <iostream> #include <iostream>
#include <numeric> #include <numeric>
...@@ -80,6 +85,55 @@ bool ReadBinaryFile(std::vector<unsigned char> *data, ...@@ -80,6 +85,55 @@ bool ReadBinaryFile(std::vector<unsigned char> *data,
return true; return true;
} }
bool MemoryMap(const std::string &file,
const unsigned char **data,
size_t *size) {
bool ret = true;
int fd = open(file.c_str(), O_RDONLY);
if (fd < 0) {
std::cerr << "Failed to open file " << file
<< ", error code: " << strerror(errno) << std::endl;
ret = false;
}
struct stat st;
fstat(fd, &st);
*size = static_cast<size_t>(st.st_size);
*data = static_cast<const unsigned char *>(
mmap(nullptr, *size, PROT_READ, MAP_PRIVATE, fd, 0));
if (*data == static_cast<const unsigned char *>(MAP_FAILED)) {
std::cerr << "Failed to map file " << file
<< ", error code: " << strerror(errno) << std::endl;
ret = false;
}
if (close(fd) < 0) {
std::cerr << "Failed to close file " << file
<< ", error code: " << strerror(errno) << std::endl;
ret = false;
}
return ret;
}
bool MemoryUnMap(const unsigned char *data,
const size_t &size) {
bool ret = true;
if (data == nullptr || size == 0) {
std::cerr << "data is null or size is 0" << std::endl;
ret = false;
}
if (munmap(const_cast<unsigned char *>(data), size) < 0) {
std::cerr << "Failed to unmap file, error code: "
<< strerror(errno) << std::endl;
ret = false;
}
return ret;
}
} // namespace } // namespace
void ParseShape(const std::string &str, std::vector<int64_t> *shape) { void ParseShape(const std::string &str, std::vector<int64_t> *shape) {
...@@ -178,7 +232,7 @@ bool RunModel(const std::vector<std::string> &input_names, ...@@ -178,7 +232,7 @@ bool RunModel(const std::vector<std::string> &input_names,
MaceEngineConfig config(device_type); MaceEngineConfig config(device_type);
status = config.SetCPUThreadPolicy( status = config.SetCPUThreadPolicy(
FLAGS_omp_num_threads, FLAGS_omp_num_threads,
static_cast<CPUAffinityPolicy >(FLAGS_cpu_affinity_policy)); static_cast<CPUAffinityPolicy>(FLAGS_cpu_affinity_policy));
if (status != MaceStatus::MACE_SUCCESS) { if (status != MaceStatus::MACE_SUCCESS) {
std::cerr << "Set openmp or cpu affinity failed." << std::endl; std::cerr << "Set openmp or cpu affinity failed." << std::endl;
} }
...@@ -216,8 +270,11 @@ bool RunModel(const std::vector<std::string> &input_names, ...@@ -216,8 +270,11 @@ bool RunModel(const std::vector<std::string> &input_names,
if (!ReadBinaryFile(&model_graph_data, FLAGS_model_file)) { if (!ReadBinaryFile(&model_graph_data, FLAGS_model_file)) {
std::cerr << "Failed to read file: " << FLAGS_model_file << std::endl; std::cerr << "Failed to read file: " << FLAGS_model_file << std::endl;
} }
std::vector<unsigned char> model_weights_data; const unsigned char *model_weights_data = nullptr;
if (!ReadBinaryFile(&model_weights_data, FLAGS_model_data_file)) { size_t model_weights_data_size = 0;
if (!MemoryMap(FLAGS_model_data_file,
&model_weights_data,
&model_weights_data_size)) {
std::cerr << "Failed to read file: " << FLAGS_model_data_file << std::endl; std::cerr << "Failed to read file: " << FLAGS_model_data_file << std::endl;
} }
...@@ -238,8 +295,8 @@ bool RunModel(const std::vector<std::string> &input_names, ...@@ -238,8 +295,8 @@ bool RunModel(const std::vector<std::string> &input_names,
create_engine_status = create_engine_status =
CreateMaceEngineFromProto(model_graph_data.data(), CreateMaceEngineFromProto(model_graph_data.data(),
model_graph_data.size(), model_graph_data.size(),
model_weights_data.data(), model_weights_data,
model_weights_data.size(), model_weights_data_size,
input_names, input_names,
output_names, output_names,
config, config,
...@@ -373,6 +430,10 @@ bool RunModel(const std::vector<std::string> &input_names, ...@@ -373,6 +430,10 @@ bool RunModel(const std::vector<std::string> &input_names,
} }
} }
if (model_weights_data != nullptr) {
MemoryUnMap(model_weights_data, model_weights_data_size);
}
std::cout << "Finished" << std::endl; std::cout << "Finished" << std::endl;
return true; return true;
......
...@@ -12,12 +12,6 @@ ...@@ -12,12 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>
#include <algorithm> #include <algorithm>
#include <numeric> #include <numeric>
#include <memory> #include <memory>
...@@ -41,33 +35,6 @@ ...@@ -41,33 +35,6 @@
namespace mace { namespace mace {
namespace { namespace {
const unsigned char *LoadModelData(const std::string &model_data_file,
const size_t &data_size) {
int fd = open(model_data_file.c_str(), O_RDONLY);
MACE_CHECK(fd >= 0, "Failed to open model data file ",
model_data_file, ", error code: ", strerror(errno));
const unsigned char *model_data = static_cast<const unsigned char *>(
mmap(nullptr, data_size, PROT_READ, MAP_PRIVATE, fd, 0));
MACE_CHECK(model_data != MAP_FAILED, "Failed to map model data file ",
model_data_file, ", error code: ", strerror(errno));
int ret = close(fd);
MACE_CHECK(ret == 0, "Failed to close model data file ",
model_data_file, ", error code: ", strerror(errno));
return model_data;
}
void UnloadModelData(const unsigned char *model_data,
const size_t &data_size) {
MACE_CHECK(model_data != nullptr && data_size > 0,
"model_data is null or data_size is 0");
int ret = munmap(const_cast<unsigned char *>(model_data),
data_size);
MACE_CHECK(ret == 0, "Failed to unmap model data file, error code: ",
strerror(errno));
}
#ifdef MACE_ENABLE_OPENCL #ifdef MACE_ENABLE_OPENCL
MaceStatus CheckGPUAvalibility(const NetDef *net_def, Device *device) { MaceStatus CheckGPUAvalibility(const NetDef *net_def, Device *device) {
...@@ -557,20 +524,14 @@ MaceStatus MaceEngine::Impl::Init( ...@@ -557,20 +524,14 @@ MaceStatus MaceEngine::Impl::Init(
const std::vector<std::string> &output_nodes, const std::vector<std::string> &output_nodes,
const std::string &model_data_file) { const std::string &model_data_file) {
LOG(INFO) << "Loading Model Data"; LOG(INFO) << "Loading Model Data";
for (auto &const_tensor : net_def->tensors()) {
model_data_size_ = std::max( MemoryMap(model_data_file, &model_data_, &model_data_size_);
model_data_size_,
static_cast<size_t>(const_tensor.offset() +
const_tensor.data_size() *
GetEnumTypeSize(const_tensor.data_type())));
}
model_data_ = LoadModelData(model_data_file, model_data_size_);
MACE_RETURN_IF_ERROR(Init(net_def, input_nodes, output_nodes, model_data_)); MACE_RETURN_IF_ERROR(Init(net_def, input_nodes, output_nodes, model_data_));
if (device_type_ == DeviceType::GPU || device_type_ == DeviceType::HEXAGON || if (device_type_ == DeviceType::GPU || device_type_ == DeviceType::HEXAGON ||
(device_type_ == DeviceType::CPU && ws_->diffused_buffer())) { (device_type_ == DeviceType::CPU && ws_->diffused_buffer())) {
UnloadModelData(model_data_, model_data_size_); MemoryUnMap(model_data_, model_data_size_);
model_data_ = nullptr; model_data_ = nullptr;
} }
return MaceStatus::MACE_SUCCESS; return MaceStatus::MACE_SUCCESS;
...@@ -579,7 +540,7 @@ MaceStatus MaceEngine::Impl::Init( ...@@ -579,7 +540,7 @@ MaceStatus MaceEngine::Impl::Init(
MaceEngine::Impl::~Impl() { MaceEngine::Impl::~Impl() {
LOG(INFO) << "Destroying MaceEngine"; LOG(INFO) << "Destroying MaceEngine";
if (model_data_ != nullptr) { if (model_data_ != nullptr) {
UnloadModelData(model_data_, model_data_size_); MemoryUnMap(model_data_, model_data_size_);
} }
#ifdef MACE_ENABLE_HEXAGON #ifdef MACE_ENABLE_HEXAGON
if (device_type_ == HEXAGON) { if (device_type_ == HEXAGON) {
......
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
#include <stdint.h> #include <stdint.h>
#include <cstdio> #include <cstdio>
#include <cstdlib> #include <cstdlib>
#include <fstream>
#include <iostream> #include <iostream>
#include <numeric> #include <numeric>
...@@ -208,6 +209,8 @@ bool RunModel(const std::string &model_name, ...@@ -208,6 +209,8 @@ bool RunModel(const std::string &model_name,
const std::vector<std::string> &output_names, const std::vector<std::string> &output_names,
const std::vector<std::vector<int64_t>> &output_shapes) { const std::vector<std::vector<int64_t>> &output_shapes) {
DeviceType device_type = ParseDeviceType(FLAGS_device); DeviceType device_type = ParseDeviceType(FLAGS_device);
int64_t t0 = NowMicros();
// config runtime // config runtime
MaceStatus status; MaceStatus status;
MaceEngineConfig config(device_type); MaceEngineConfig config(device_type);
...@@ -247,17 +250,18 @@ bool RunModel(const std::string &model_name, ...@@ -247,17 +250,18 @@ bool RunModel(const std::string &model_name,
} }
} }
std::vector<unsigned char> model_weights_data; const unsigned char *model_weights_data = nullptr;
size_t model_weights_data_size = 0;
if (FLAGS_model_data_file != "") { if (FLAGS_model_data_file != "") {
if (!mace::ReadBinaryFile(&model_weights_data, FLAGS_model_data_file)) { MemoryMap(FLAGS_model_data_file,
LOG(FATAL) << "Failed to read file: " << FLAGS_model_data_file; &model_weights_data,
} &model_weights_data_size);
MACE_CHECK(model_weights_data != nullptr && model_weights_data_size != 0);
} }
std::shared_ptr<mace::MaceEngine> engine; std::shared_ptr<mace::MaceEngine> engine;
MaceStatus create_engine_status; MaceStatus create_engine_status;
double init_millis;
while (true) { while (true) {
// Create Engine // Create Engine
int64_t t0 = NowMicros(); int64_t t0 = NowMicros();
...@@ -275,8 +279,8 @@ bool RunModel(const std::string &model_name, ...@@ -275,8 +279,8 @@ bool RunModel(const std::string &model_name,
create_engine_status = create_engine_status =
CreateMaceEngineFromProto(model_graph_data.data(), CreateMaceEngineFromProto(model_graph_data.data(),
model_graph_data.size(), model_graph_data.size(),
model_weights_data.data(), model_weights_data,
model_weights_data.size(), model_weights_data_size,
input_names, input_names,
output_names, output_names,
config, config,
...@@ -288,11 +292,15 @@ bool RunModel(const std::string &model_name, ...@@ -288,11 +292,15 @@ bool RunModel(const std::string &model_name,
LOG(ERROR) << "Create engine runtime error, retry ... errcode: " LOG(ERROR) << "Create engine runtime error, retry ... errcode: "
<< create_engine_status.information(); << create_engine_status.information();
} else { } else {
init_millis = (t1 - t0) / 1000.0; double create_engine_millis = (t1 - t0) / 1000.0;
LOG(INFO) << "Total init latency: " << init_millis << " ms"; LOG(INFO) << "Create Mace Engine latency: " << create_engine_millis
<< " ms";
break; break;
} }
} }
int64_t t1 = NowMicros();
double init_millis = (t1 - t0) / 1000.0;
LOG(INFO) << "Total init latency: " << init_millis << " ms";
const size_t input_count = input_names.size(); const size_t input_count = input_names.size();
const size_t output_count = output_names.size(); const size_t output_count = output_names.size();
...@@ -351,8 +359,8 @@ bool RunModel(const std::string &model_name, ...@@ -351,8 +359,8 @@ bool RunModel(const std::string &model_name,
create_engine_status = create_engine_status =
CreateMaceEngineFromProto(model_graph_data.data(), CreateMaceEngineFromProto(model_graph_data.data(),
model_graph_data.size(), model_graph_data.size(),
model_weights_data.data(), model_weights_data,
model_weights_data.size(), model_weights_data_size,
input_names, input_names,
output_names, output_names,
config, config,
...@@ -394,8 +402,8 @@ bool RunModel(const std::string &model_name, ...@@ -394,8 +402,8 @@ bool RunModel(const std::string &model_name,
create_engine_status = create_engine_status =
CreateMaceEngineFromProto(model_graph_data.data(), CreateMaceEngineFromProto(model_graph_data.data(),
model_graph_data.size(), model_graph_data.size(),
model_weights_data.data(), model_weights_data,
model_weights_data.size(), model_weights_data_size,
input_names, input_names,
output_names, output_names,
config, config,
...@@ -441,6 +449,10 @@ bool RunModel(const std::string &model_name, ...@@ -441,6 +449,10 @@ bool RunModel(const std::string &model_name,
<< output_size << " done."; << output_size << " done.";
} }
if (model_weights_data != nullptr) {
MemoryUnMap(model_weights_data, model_weights_data_size);
}
return true; return true;
} }
......
...@@ -7,14 +7,22 @@ package( ...@@ -7,14 +7,22 @@ package(
licenses(["notice"]) # Apache 2.0 licenses(["notice"]) # Apache 2.0
load("//mace:mace.bzl", "if_android") load(
"//mace:mace.bzl",
"if_android",
"if_openmp_enabled",
)
cc_library( cc_library(
name = "utils", name = "utils",
srcs = [ srcs = glob(
"logging.cc", [
"string_util.cc", "*.cc",
], ],
exclude = [
"tuner_test.cc",
],
),
hdrs = glob([ hdrs = glob([
"*.h", "*.h",
]), ]),
...@@ -22,7 +30,9 @@ cc_library( ...@@ -22,7 +30,9 @@ cc_library(
"-Werror", "-Werror",
"-Wextra", "-Wextra",
"-Wno-missing-field-initializers", "-Wno-missing-field-initializers",
], ] + if_openmp_enabled([
"-fopenmp",
]),
linkopts = if_android([ linkopts = if_android([
"-llog", "-llog",
]), ]),
......
// Copyright 2018 The MACE Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <fstream>
#include <map>
#include <string>
#include <unordered_map>
#include <vector>
namespace mace {
bool GetTuningParams(
const char *path,
std::unordered_map<std::string, std::vector<unsigned int>> *param_table) {
if (path != nullptr) {
std::ifstream ifs(path, std::ios::binary | std::ios::in);
if (ifs.is_open()) {
int64_t num_params = 0;
ifs.read(reinterpret_cast<char *>(&num_params), sizeof(num_params));
while (num_params--) {
int32_t key_size = 0;
ifs.read(reinterpret_cast<char *>(&key_size), sizeof(key_size));
std::string key(key_size, ' ');
ifs.read(&key[0], key_size);
int32_t params_size = 0;
ifs.read(reinterpret_cast<char *>(&params_size), sizeof(params_size));
int32_t params_count = params_size / sizeof(unsigned int);
std::vector<unsigned int> params(params_count);
for (int i = 0; i < params_count; ++i) {
ifs.read(reinterpret_cast<char *>(&params[i]), sizeof(unsigned int));
}
param_table->emplace(key, params);
}
ifs.close();
} else {
return false;
}
}
return true;
}
} // namespace mace
// Copyright 2018 The MACE Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <map>
#include <string>
#include <unordered_map>
#include <vector>
namespace mace {
bool GetTuningParams(
const char *path,
std::unordered_map<std::string, std::vector<unsigned int>> *param_table) {
(void)(path);
extern const std::map<std::string, std::vector<unsigned int>>
kTuningParamsData;
for (auto it = kTuningParamsData.begin(); it != kTuningParamsData.end();
++it) {
param_table->emplace(it->first, std::vector<unsigned int>(
it->second.begin(), it->second.end()));
}
return true;
}
} // namespace mace
// Copyright 2018 The MACE Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mace/utils/utils.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <cerrno>
#include <cstring>
#include <fstream>
#include <map>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#include "mace/utils/logging.h"
namespace mace {
std::string ObfuscateString(const std::string &src,
const std::string &lookup_table) {
std::string dest;
dest.resize(src.size());
for (size_t i = 0; i < src.size(); i++) {
dest[i] = src[i] ^ lookup_table[i % lookup_table.size()];
}
return dest;
}
// ObfuscateString(ObfuscateString(str)) ==> str
std::string ObfuscateString(const std::string &src) {
// Keep consistent with obfuscation in python tools
return ObfuscateString(src, "Mobile-AI-Compute-Engine");
}
// Obfuscate synbol or path string
std::string ObfuscateSymbol(const std::string &src) {
std::string dest = src;
if (dest.empty()) {
return dest;
}
dest[0] = src[0]; // avoid invalid symbol which starts from 0-9
const std::string encode_dict =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_";
for (size_t i = 1; i < src.size(); i++) {
char ch = src[i];
int idx;
if (ch >= '0' && ch <= '9') {
idx = ch - '0';
} else if (ch >= 'a' && ch <= 'z') {
idx = 10 + ch - 'a';
} else if (ch >= 'A' && ch <= 'Z') {
idx = 10 + 26 + ch - 'a';
} else if (ch == '_') {
idx = 10 + 26 + 26;
} else {
dest[i] = ch;
continue;
}
// There is no collision if it's true for every char at every position
dest[i] = encode_dict[(idx + i + 31) % encode_dict.size()];
}
return dest;
}
std::vector<std::string> Split(const std::string &str, char delims) {
std::vector<std::string> result;
std::string tmp = str;
while (!tmp.empty()) {
size_t next_offset = tmp.find(delims);
result.push_back(tmp.substr(0, next_offset));
if (next_offset == std::string::npos) {
break;
} else {
tmp = tmp.substr(next_offset + 1);
}
}
return result;
}
bool ReadBinaryFile(std::vector<unsigned char> *data,
const std::string &filename) {
std::ifstream ifs(filename, std::ios::in | std::ios::binary);
if (!ifs.is_open()) {
return false;
}
ifs.seekg(0, ifs.end);
size_t length = ifs.tellg();
ifs.seekg(0, ifs.beg);
data->resize(length);
ifs.read(reinterpret_cast<char *>(data->data()), length);
if (ifs.fail()) {
return false;
}
ifs.close();
return true;
}
void MemoryMap(const std::string &file,
const unsigned char **data,
size_t *size) {
int fd = open(file.c_str(), O_RDONLY);
MACE_CHECK(fd >= 0,
"Failed to open file ", file, ", error code: ", strerror(errno));
struct stat st;
fstat(fd, &st);
*size = static_cast<size_t>(st.st_size);
*data = static_cast<const unsigned char *>(
mmap(nullptr, *size, PROT_READ, MAP_PRIVATE, fd, 0));
MACE_CHECK(*data != static_cast<const unsigned char *>(MAP_FAILED),
"Failed to map file ", file, ", error code: ", strerror(errno));
int ret = close(fd);
MACE_CHECK(ret == 0,
"Failed to close file ", file, ", error code: ", strerror(errno));
}
void MemoryUnMap(const unsigned char *data,
const size_t &size) {
MACE_CHECK(data != nullptr && size > 0, "data is null or size is 0");
int ret = munmap(const_cast<unsigned char *>(data), size);
MACE_CHECK(ret == 0,
"Failed to unmap file, error code: ", strerror(errno));
}
} // namespace mace
...@@ -15,12 +15,9 @@ ...@@ -15,12 +15,9 @@
#ifndef MACE_UTILS_UTILS_H_ #ifndef MACE_UTILS_UTILS_H_
#define MACE_UTILS_UTILS_H_ #define MACE_UTILS_UTILS_H_
#include <fstream> #include <cstdlib>
#include <map> #include <map>
#include <sstream>
#include <string> #include <string>
#include <cstdlib>
#include <utility>
#include <vector> #include <vector>
namespace mace { namespace mace {
...@@ -70,51 +67,12 @@ Integer CeilQuotient(Integer a, Integer b) { ...@@ -70,51 +67,12 @@ Integer CeilQuotient(Integer a, Integer b) {
return (a + b - 1) / b; return (a + b - 1) / b;
} }
inline std::string ObfuscateString(const std::string &src, std::string ObfuscateString(const std::string &src,
const std::string &lookup_table) { const std::string &lookup_table);
std::string dest;
dest.resize(src.size());
for (size_t i = 0; i < src.size(); i++) {
dest[i] = src[i] ^ lookup_table[i % lookup_table.size()];
}
return dest;
}
// ObfuscateString(ObfuscateString(str)) ==> str std::string ObfuscateString(const std::string &src);
inline std::string ObfuscateString(const std::string &src) {
// Keep consistent with obfuscation in python tools
return ObfuscateString(src, "Mobile-AI-Compute-Engine");
}
// Obfuscate synbol or path string std::string ObfuscateSymbol(const std::string &src);
inline std::string ObfuscateSymbol(const std::string &src) {
std::string dest = src;
if (dest.empty()) {
return dest;
}
dest[0] = src[0]; // avoid invalid symbol which starts from 0-9
const std::string encode_dict =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_";
for (size_t i = 1; i < src.size(); i++) {
char ch = src[i];
int idx;
if (ch >= '0' && ch <= '9') {
idx = ch - '0';
} else if (ch >= 'a' && ch <= 'z') {
idx = 10 + ch - 'a';
} else if (ch >= 'A' && ch <= 'Z') {
idx = 10 + 26 + ch - 'a';
} else if (ch == '_') {
idx = 10 + 26 + 26;
} else {
dest[i] = ch;
continue;
}
// There is no collision if it's true for every char at every position
dest[i] = encode_dict[(idx + i + 31) % encode_dict.size()];
}
return dest;
}
#ifdef MACE_OBFUSCATE_LITERALS #ifdef MACE_OBFUSCATE_LITERALS
#define MACE_OBFUSCATE_STRING(str) ObfuscateString(str) #define MACE_OBFUSCATE_STRING(str) ObfuscateString(str)
...@@ -124,41 +82,17 @@ inline std::string ObfuscateSymbol(const std::string &src) { ...@@ -124,41 +82,17 @@ inline std::string ObfuscateSymbol(const std::string &src) {
#define MACE_OBFUSCATE_SYMBOL(str) (str) #define MACE_OBFUSCATE_SYMBOL(str) (str)
#endif #endif
inline std::vector<std::string> Split(const std::string &str, char delims) { std::vector<std::string> Split(const std::string &str, char delims);
std::vector<std::string> result;
std::string tmp = str;
while (!tmp.empty()) {
size_t next_offset = tmp.find(delims);
result.push_back(tmp.substr(0, next_offset));
if (next_offset == std::string::npos) {
break;
} else {
tmp = tmp.substr(next_offset + 1);
}
}
return result;
}
inline bool ReadBinaryFile(std::vector<unsigned char> *data, bool ReadBinaryFile(std::vector<unsigned char> *data,
const std::string &filename) { const std::string &filename);
std::ifstream ifs(filename, std::ios::in | std::ios::binary);
if (!ifs.is_open()) {
return false;
}
ifs.seekg(0, ifs.end);
size_t length = ifs.tellg();
ifs.seekg(0, ifs.beg);
data->reserve(length);
data->insert(data->begin(), std::istreambuf_iterator<char>(ifs),
std::istreambuf_iterator<char>());
if (ifs.fail()) {
return false;
}
ifs.close();
return true; void MemoryMap(const std::string &file,
} const unsigned char **data,
size_t *size);
void MemoryUnMap(const unsigned char *data,
const size_t &size);
template <typename T> template <typename T>
std::vector<std::string> MapKeys(const std::map<std::string, T> &data) { std::vector<std::string> MapKeys(const std::map<std::string, T> &data) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册