Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
毕竟曾有刹那
Mace
提交
3beb766d
Mace
项目概览
毕竟曾有刹那
/
Mace
与 Fork 源项目一致
Fork自
Xiaomi / Mace
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
DevOps
流水线
流水线任务
计划
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
Mace
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
DevOps
DevOps
流水线
流水线任务
计划
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
流水线任务
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3beb766d
编写于
2月 15, 2019
作者:
李
李寅
浏览文件
操作
浏览文件
下载
差异文件
Merge branch 'optimize' into 'master'
Optimize load model data See merge request !981
上级
e8d5b186
51172108
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
285 addition
and
244 deletion
+285
-244
mace/benchmark/benchmark_model.cc
mace/benchmark/benchmark_model.cc
+12
-6
mace/core/runtime/cpu/cpu_runtime.cc
mace/core/runtime/cpu/cpu_runtime.cc
+1
-0
mace/examples/cli/example.cc
mace/examples/cli/example.cc
+67
-6
mace/libmace/mace.cc
mace/libmace/mace.cc
+4
-43
mace/tools/validation/mace_run.cc
mace/tools/validation/mace_run.cc
+25
-13
mace/utils/BUILD
mace/utils/BUILD
+16
-6
mace/utils/tuner_development.cc
mace/utils/tuner_development.cc
+0
-54
mace/utils/tuner_production.cc
mace/utils/tuner_production.cc
+0
-36
mace/utils/utils.cc
mace/utils/utils.cc
+146
-0
mace/utils/utils.h
mace/utils/utils.h
+14
-80
未找到文件。
mace/benchmark/benchmark_model.cc
浏览文件 @
3beb766d
...
...
@@ -280,11 +280,13 @@ int Main(int argc, char **argv) {
}
}
std
::
vector
<
unsigned
char
>
model_weights_data
;
const
unsigned
char
*
model_weights_data
=
nullptr
;
size_t
model_weights_data_size
=
0
;
if
(
FLAGS_model_data_file
!=
""
)
{
if
(
!
mace
::
ReadBinaryFile
(
&
model_weights_data
,
FLAGS_model_data_file
))
{
LOG
(
FATAL
)
<<
"Failed to read file: "
<<
FLAGS_model_data_file
;
}
MemoryMap
(
FLAGS_model_data_file
,
&
model_weights_data
,
&
model_weights_data_size
);
MACE_CHECK
(
model_weights_data
!=
nullptr
&&
model_weights_data_size
!=
0
);
}
#ifdef MODEL_GRAPH_FORMAT_CODE
...
...
@@ -300,8 +302,8 @@ int Main(int argc, char **argv) {
create_engine_status
=
CreateMaceEngineFromProto
(
model_graph_data
.
data
(),
model_graph_data
.
size
(),
model_weights_data
.
data
()
,
model_weights_data
.
size
()
,
model_weights_data
,
model_weights_data
_size
,
input_names
,
output_names
,
config
,
...
...
@@ -378,6 +380,10 @@ int Main(int argc, char **argv) {
statistician
->
PrintStat
();
if
(
model_weights_data
!=
nullptr
)
{
MemoryUnMap
(
model_weights_data
,
model_weights_data_size
);
}
return
0
;
}
...
...
mace/core/runtime/cpu/cpu_runtime.cc
浏览文件 @
3beb766d
...
...
@@ -24,6 +24,7 @@
#include <algorithm>
#include <cerrno>
#include <cstring>
#include <fstream>
#include <string>
#include <utility>
#include <vector>
...
...
mace/examples/cli/example.cc
浏览文件 @
3beb766d
...
...
@@ -13,9 +13,14 @@
// limitations under the License.
#include <dirent.h>
#include <fcntl.h>
#include <malloc.h>
#include <stdint.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <iostream>
#include <numeric>
...
...
@@ -80,6 +85,55 @@ bool ReadBinaryFile(std::vector<unsigned char> *data,
return
true
;
}
bool
MemoryMap
(
const
std
::
string
&
file
,
const
unsigned
char
**
data
,
size_t
*
size
)
{
bool
ret
=
true
;
int
fd
=
open
(
file
.
c_str
(),
O_RDONLY
);
if
(
fd
<
0
)
{
std
::
cerr
<<
"Failed to open file "
<<
file
<<
", error code: "
<<
strerror
(
errno
)
<<
std
::
endl
;
ret
=
false
;
}
struct
stat
st
;
fstat
(
fd
,
&
st
);
*
size
=
static_cast
<
size_t
>
(
st
.
st_size
);
*
data
=
static_cast
<
const
unsigned
char
*>
(
mmap
(
nullptr
,
*
size
,
PROT_READ
,
MAP_PRIVATE
,
fd
,
0
));
if
(
*
data
==
static_cast
<
const
unsigned
char
*>
(
MAP_FAILED
))
{
std
::
cerr
<<
"Failed to map file "
<<
file
<<
", error code: "
<<
strerror
(
errno
)
<<
std
::
endl
;
ret
=
false
;
}
if
(
close
(
fd
)
<
0
)
{
std
::
cerr
<<
"Failed to close file "
<<
file
<<
", error code: "
<<
strerror
(
errno
)
<<
std
::
endl
;
ret
=
false
;
}
return
ret
;
}
bool
MemoryUnMap
(
const
unsigned
char
*
data
,
const
size_t
&
size
)
{
bool
ret
=
true
;
if
(
data
==
nullptr
||
size
==
0
)
{
std
::
cerr
<<
"data is null or size is 0"
<<
std
::
endl
;
ret
=
false
;
}
if
(
munmap
(
const_cast
<
unsigned
char
*>
(
data
),
size
)
<
0
)
{
std
::
cerr
<<
"Failed to unmap file, error code: "
<<
strerror
(
errno
)
<<
std
::
endl
;
ret
=
false
;
}
return
ret
;
}
}
// namespace
void
ParseShape
(
const
std
::
string
&
str
,
std
::
vector
<
int64_t
>
*
shape
)
{
...
...
@@ -178,7 +232,7 @@ bool RunModel(const std::vector<std::string> &input_names,
MaceEngineConfig
config
(
device_type
);
status
=
config
.
SetCPUThreadPolicy
(
FLAGS_omp_num_threads
,
static_cast
<
CPUAffinityPolicy
>
(
FLAGS_cpu_affinity_policy
));
static_cast
<
CPUAffinityPolicy
>
(
FLAGS_cpu_affinity_policy
));
if
(
status
!=
MaceStatus
::
MACE_SUCCESS
)
{
std
::
cerr
<<
"Set openmp or cpu affinity failed."
<<
std
::
endl
;
}
...
...
@@ -216,8 +270,11 @@ bool RunModel(const std::vector<std::string> &input_names,
if
(
!
ReadBinaryFile
(
&
model_graph_data
,
FLAGS_model_file
))
{
std
::
cerr
<<
"Failed to read file: "
<<
FLAGS_model_file
<<
std
::
endl
;
}
std
::
vector
<
unsigned
char
>
model_weights_data
;
if
(
!
ReadBinaryFile
(
&
model_weights_data
,
FLAGS_model_data_file
))
{
const
unsigned
char
*
model_weights_data
=
nullptr
;
size_t
model_weights_data_size
=
0
;
if
(
!
MemoryMap
(
FLAGS_model_data_file
,
&
model_weights_data
,
&
model_weights_data_size
))
{
std
::
cerr
<<
"Failed to read file: "
<<
FLAGS_model_data_file
<<
std
::
endl
;
}
...
...
@@ -238,8 +295,8 @@ bool RunModel(const std::vector<std::string> &input_names,
create_engine_status
=
CreateMaceEngineFromProto
(
model_graph_data
.
data
(),
model_graph_data
.
size
(),
model_weights_data
.
data
()
,
model_weights_data
.
size
()
,
model_weights_data
,
model_weights_data
_size
,
input_names
,
output_names
,
config
,
...
...
@@ -373,6 +430,10 @@ bool RunModel(const std::vector<std::string> &input_names,
}
}
if
(
model_weights_data
!=
nullptr
)
{
MemoryUnMap
(
model_weights_data
,
model_weights_data_size
);
}
std
::
cout
<<
"Finished"
<<
std
::
endl
;
return
true
;
...
...
mace/libmace/mace.cc
浏览文件 @
3beb766d
...
...
@@ -12,12 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>
#include <algorithm>
#include <numeric>
#include <memory>
...
...
@@ -41,33 +35,6 @@
namespace
mace
{
namespace
{
const
unsigned
char
*
LoadModelData
(
const
std
::
string
&
model_data_file
,
const
size_t
&
data_size
)
{
int
fd
=
open
(
model_data_file
.
c_str
(),
O_RDONLY
);
MACE_CHECK
(
fd
>=
0
,
"Failed to open model data file "
,
model_data_file
,
", error code: "
,
strerror
(
errno
));
const
unsigned
char
*
model_data
=
static_cast
<
const
unsigned
char
*>
(
mmap
(
nullptr
,
data_size
,
PROT_READ
,
MAP_PRIVATE
,
fd
,
0
));
MACE_CHECK
(
model_data
!=
MAP_FAILED
,
"Failed to map model data file "
,
model_data_file
,
", error code: "
,
strerror
(
errno
));
int
ret
=
close
(
fd
);
MACE_CHECK
(
ret
==
0
,
"Failed to close model data file "
,
model_data_file
,
", error code: "
,
strerror
(
errno
));
return
model_data
;
}
void
UnloadModelData
(
const
unsigned
char
*
model_data
,
const
size_t
&
data_size
)
{
MACE_CHECK
(
model_data
!=
nullptr
&&
data_size
>
0
,
"model_data is null or data_size is 0"
);
int
ret
=
munmap
(
const_cast
<
unsigned
char
*>
(
model_data
),
data_size
);
MACE_CHECK
(
ret
==
0
,
"Failed to unmap model data file, error code: "
,
strerror
(
errno
));
}
#ifdef MACE_ENABLE_OPENCL
MaceStatus
CheckGPUAvalibility
(
const
NetDef
*
net_def
,
Device
*
device
)
{
...
...
@@ -557,20 +524,14 @@ MaceStatus MaceEngine::Impl::Init(
const
std
::
vector
<
std
::
string
>
&
output_nodes
,
const
std
::
string
&
model_data_file
)
{
LOG
(
INFO
)
<<
"Loading Model Data"
;
for
(
auto
&
const_tensor
:
net_def
->
tensors
())
{
model_data_size_
=
std
::
max
(
model_data_size_
,
static_cast
<
size_t
>
(
const_tensor
.
offset
()
+
const_tensor
.
data_size
()
*
GetEnumTypeSize
(
const_tensor
.
data_type
())));
}
model_data_
=
LoadModelData
(
model_data_file
,
model_data_size_
);
MemoryMap
(
model_data_file
,
&
model_data_
,
&
model_data_size_
);
MACE_RETURN_IF_ERROR
(
Init
(
net_def
,
input_nodes
,
output_nodes
,
model_data_
));
if
(
device_type_
==
DeviceType
::
GPU
||
device_type_
==
DeviceType
::
HEXAGON
||
(
device_type_
==
DeviceType
::
CPU
&&
ws_
->
diffused_buffer
()))
{
UnloadModelData
(
model_data_
,
model_data_size_
);
MemoryUnMap
(
model_data_
,
model_data_size_
);
model_data_
=
nullptr
;
}
return
MaceStatus
::
MACE_SUCCESS
;
...
...
@@ -579,7 +540,7 @@ MaceStatus MaceEngine::Impl::Init(
MaceEngine
::
Impl
::~
Impl
()
{
LOG
(
INFO
)
<<
"Destroying MaceEngine"
;
if
(
model_data_
!=
nullptr
)
{
UnloadModelData
(
model_data_
,
model_data_size_
);
MemoryUnMap
(
model_data_
,
model_data_size_
);
}
#ifdef MACE_ENABLE_HEXAGON
if
(
device_type_
==
HEXAGON
)
{
...
...
mace/tools/validation/mace_run.cc
浏览文件 @
3beb766d
...
...
@@ -28,6 +28,7 @@
#include <stdint.h>
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <numeric>
...
...
@@ -208,6 +209,8 @@ bool RunModel(const std::string &model_name,
const
std
::
vector
<
std
::
string
>
&
output_names
,
const
std
::
vector
<
std
::
vector
<
int64_t
>>
&
output_shapes
)
{
DeviceType
device_type
=
ParseDeviceType
(
FLAGS_device
);
int64_t
t0
=
NowMicros
();
// config runtime
MaceStatus
status
;
MaceEngineConfig
config
(
device_type
);
...
...
@@ -247,17 +250,18 @@ bool RunModel(const std::string &model_name,
}
}
std
::
vector
<
unsigned
char
>
model_weights_data
;
const
unsigned
char
*
model_weights_data
=
nullptr
;
size_t
model_weights_data_size
=
0
;
if
(
FLAGS_model_data_file
!=
""
)
{
if
(
!
mace
::
ReadBinaryFile
(
&
model_weights_data
,
FLAGS_model_data_file
))
{
LOG
(
FATAL
)
<<
"Failed to read file: "
<<
FLAGS_model_data_file
;
}
MemoryMap
(
FLAGS_model_data_file
,
&
model_weights_data
,
&
model_weights_data_size
);
MACE_CHECK
(
model_weights_data
!=
nullptr
&&
model_weights_data_size
!=
0
);
}
std
::
shared_ptr
<
mace
::
MaceEngine
>
engine
;
MaceStatus
create_engine_status
;
double
init_millis
;
while
(
true
)
{
// Create Engine
int64_t
t0
=
NowMicros
();
...
...
@@ -275,8 +279,8 @@ bool RunModel(const std::string &model_name,
create_engine_status
=
CreateMaceEngineFromProto
(
model_graph_data
.
data
(),
model_graph_data
.
size
(),
model_weights_data
.
data
()
,
model_weights_data
.
size
()
,
model_weights_data
,
model_weights_data
_size
,
input_names
,
output_names
,
config
,
...
...
@@ -288,11 +292,15 @@ bool RunModel(const std::string &model_name,
LOG
(
ERROR
)
<<
"Create engine runtime error, retry ... errcode: "
<<
create_engine_status
.
information
();
}
else
{
init_millis
=
(
t1
-
t0
)
/
1000.0
;
LOG
(
INFO
)
<<
"Total init latency: "
<<
init_millis
<<
" ms"
;
double
create_engine_millis
=
(
t1
-
t0
)
/
1000.0
;
LOG
(
INFO
)
<<
"Create Mace Engine latency: "
<<
create_engine_millis
<<
" ms"
;
break
;
}
}
int64_t
t1
=
NowMicros
();
double
init_millis
=
(
t1
-
t0
)
/
1000.0
;
LOG
(
INFO
)
<<
"Total init latency: "
<<
init_millis
<<
" ms"
;
const
size_t
input_count
=
input_names
.
size
();
const
size_t
output_count
=
output_names
.
size
();
...
...
@@ -351,8 +359,8 @@ bool RunModel(const std::string &model_name,
create_engine_status
=
CreateMaceEngineFromProto
(
model_graph_data
.
data
(),
model_graph_data
.
size
(),
model_weights_data
.
data
()
,
model_weights_data
.
size
()
,
model_weights_data
,
model_weights_data
_size
,
input_names
,
output_names
,
config
,
...
...
@@ -394,8 +402,8 @@ bool RunModel(const std::string &model_name,
create_engine_status
=
CreateMaceEngineFromProto
(
model_graph_data
.
data
(),
model_graph_data
.
size
(),
model_weights_data
.
data
()
,
model_weights_data
.
size
()
,
model_weights_data
,
model_weights_data
_size
,
input_names
,
output_names
,
config
,
...
...
@@ -441,6 +449,10 @@ bool RunModel(const std::string &model_name,
<<
output_size
<<
" done."
;
}
if
(
model_weights_data
!=
nullptr
)
{
MemoryUnMap
(
model_weights_data
,
model_weights_data_size
);
}
return
true
;
}
...
...
mace/utils/BUILD
浏览文件 @
3beb766d
...
...
@@ -7,14 +7,22 @@ package(
licenses
([
"notice"
])
# Apache 2.0
load
(
"//mace:mace.bzl"
,
"if_android"
)
load
(
"//mace:mace.bzl"
,
"if_android"
,
"if_openmp_enabled"
,
)
cc_library
(
name
=
"utils"
,
srcs
=
[
"logging.cc"
,
"string_util.cc"
,
],
srcs
=
glob
(
[
"*.cc"
,
],
exclude
=
[
"tuner_test.cc"
,
],
),
hdrs
=
glob
([
"*.h"
,
]),
...
...
@@ -22,7 +30,9 @@ cc_library(
"-Werror"
,
"-Wextra"
,
"-Wno-missing-field-initializers"
,
],
]
+
if_openmp_enabled
([
"-fopenmp"
,
]),
linkopts
=
if_android
([
"-llog"
,
]),
...
...
mace/utils/tuner_development.cc
已删除
100644 → 0
浏览文件 @
e8d5b186
// Copyright 2018 The MACE Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <fstream>
#include <map>
#include <string>
#include <unordered_map>
#include <vector>
namespace
mace
{
bool
GetTuningParams
(
const
char
*
path
,
std
::
unordered_map
<
std
::
string
,
std
::
vector
<
unsigned
int
>>
*
param_table
)
{
if
(
path
!=
nullptr
)
{
std
::
ifstream
ifs
(
path
,
std
::
ios
::
binary
|
std
::
ios
::
in
);
if
(
ifs
.
is_open
())
{
int64_t
num_params
=
0
;
ifs
.
read
(
reinterpret_cast
<
char
*>
(
&
num_params
),
sizeof
(
num_params
));
while
(
num_params
--
)
{
int32_t
key_size
=
0
;
ifs
.
read
(
reinterpret_cast
<
char
*>
(
&
key_size
),
sizeof
(
key_size
));
std
::
string
key
(
key_size
,
' '
);
ifs
.
read
(
&
key
[
0
],
key_size
);
int32_t
params_size
=
0
;
ifs
.
read
(
reinterpret_cast
<
char
*>
(
&
params_size
),
sizeof
(
params_size
));
int32_t
params_count
=
params_size
/
sizeof
(
unsigned
int
);
std
::
vector
<
unsigned
int
>
params
(
params_count
);
for
(
int
i
=
0
;
i
<
params_count
;
++
i
)
{
ifs
.
read
(
reinterpret_cast
<
char
*>
(
&
params
[
i
]),
sizeof
(
unsigned
int
));
}
param_table
->
emplace
(
key
,
params
);
}
ifs
.
close
();
}
else
{
return
false
;
}
}
return
true
;
}
}
// namespace mace
mace/utils/tuner_production.cc
已删除
100644 → 0
浏览文件 @
e8d5b186
// Copyright 2018 The MACE Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <map>
#include <string>
#include <unordered_map>
#include <vector>
namespace
mace
{
bool
GetTuningParams
(
const
char
*
path
,
std
::
unordered_map
<
std
::
string
,
std
::
vector
<
unsigned
int
>>
*
param_table
)
{
(
void
)(
path
);
extern
const
std
::
map
<
std
::
string
,
std
::
vector
<
unsigned
int
>>
kTuningParamsData
;
for
(
auto
it
=
kTuningParamsData
.
begin
();
it
!=
kTuningParamsData
.
end
();
++
it
)
{
param_table
->
emplace
(
it
->
first
,
std
::
vector
<
unsigned
int
>
(
it
->
second
.
begin
(),
it
->
second
.
end
()));
}
return
true
;
}
}
// namespace mace
mace/utils/utils.cc
0 → 100644
浏览文件 @
3beb766d
// Copyright 2018 The MACE Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mace/utils/utils.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#include <cerrno>
#include <cstring>
#include <fstream>
#include <map>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#include "mace/utils/logging.h"
namespace
mace
{
std
::
string
ObfuscateString
(
const
std
::
string
&
src
,
const
std
::
string
&
lookup_table
)
{
std
::
string
dest
;
dest
.
resize
(
src
.
size
());
for
(
size_t
i
=
0
;
i
<
src
.
size
();
i
++
)
{
dest
[
i
]
=
src
[
i
]
^
lookup_table
[
i
%
lookup_table
.
size
()];
}
return
dest
;
}
// ObfuscateString(ObfuscateString(str)) ==> str
std
::
string
ObfuscateString
(
const
std
::
string
&
src
)
{
// Keep consistent with obfuscation in python tools
return
ObfuscateString
(
src
,
"Mobile-AI-Compute-Engine"
);
}
// Obfuscate synbol or path string
std
::
string
ObfuscateSymbol
(
const
std
::
string
&
src
)
{
std
::
string
dest
=
src
;
if
(
dest
.
empty
())
{
return
dest
;
}
dest
[
0
]
=
src
[
0
];
// avoid invalid symbol which starts from 0-9
const
std
::
string
encode_dict
=
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"
;
for
(
size_t
i
=
1
;
i
<
src
.
size
();
i
++
)
{
char
ch
=
src
[
i
];
int
idx
;
if
(
ch
>=
'0'
&&
ch
<=
'9'
)
{
idx
=
ch
-
'0'
;
}
else
if
(
ch
>=
'a'
&&
ch
<=
'z'
)
{
idx
=
10
+
ch
-
'a'
;
}
else
if
(
ch
>=
'A'
&&
ch
<=
'Z'
)
{
idx
=
10
+
26
+
ch
-
'a'
;
}
else
if
(
ch
==
'_'
)
{
idx
=
10
+
26
+
26
;
}
else
{
dest
[
i
]
=
ch
;
continue
;
}
// There is no collision if it's true for every char at every position
dest
[
i
]
=
encode_dict
[(
idx
+
i
+
31
)
%
encode_dict
.
size
()];
}
return
dest
;
}
std
::
vector
<
std
::
string
>
Split
(
const
std
::
string
&
str
,
char
delims
)
{
std
::
vector
<
std
::
string
>
result
;
std
::
string
tmp
=
str
;
while
(
!
tmp
.
empty
())
{
size_t
next_offset
=
tmp
.
find
(
delims
);
result
.
push_back
(
tmp
.
substr
(
0
,
next_offset
));
if
(
next_offset
==
std
::
string
::
npos
)
{
break
;
}
else
{
tmp
=
tmp
.
substr
(
next_offset
+
1
);
}
}
return
result
;
}
bool
ReadBinaryFile
(
std
::
vector
<
unsigned
char
>
*
data
,
const
std
::
string
&
filename
)
{
std
::
ifstream
ifs
(
filename
,
std
::
ios
::
in
|
std
::
ios
::
binary
);
if
(
!
ifs
.
is_open
())
{
return
false
;
}
ifs
.
seekg
(
0
,
ifs
.
end
);
size_t
length
=
ifs
.
tellg
();
ifs
.
seekg
(
0
,
ifs
.
beg
);
data
->
resize
(
length
);
ifs
.
read
(
reinterpret_cast
<
char
*>
(
data
->
data
()),
length
);
if
(
ifs
.
fail
())
{
return
false
;
}
ifs
.
close
();
return
true
;
}
void
MemoryMap
(
const
std
::
string
&
file
,
const
unsigned
char
**
data
,
size_t
*
size
)
{
int
fd
=
open
(
file
.
c_str
(),
O_RDONLY
);
MACE_CHECK
(
fd
>=
0
,
"Failed to open file "
,
file
,
", error code: "
,
strerror
(
errno
));
struct
stat
st
;
fstat
(
fd
,
&
st
);
*
size
=
static_cast
<
size_t
>
(
st
.
st_size
);
*
data
=
static_cast
<
const
unsigned
char
*>
(
mmap
(
nullptr
,
*
size
,
PROT_READ
,
MAP_PRIVATE
,
fd
,
0
));
MACE_CHECK
(
*
data
!=
static_cast
<
const
unsigned
char
*>
(
MAP_FAILED
),
"Failed to map file "
,
file
,
", error code: "
,
strerror
(
errno
));
int
ret
=
close
(
fd
);
MACE_CHECK
(
ret
==
0
,
"Failed to close file "
,
file
,
", error code: "
,
strerror
(
errno
));
}
void
MemoryUnMap
(
const
unsigned
char
*
data
,
const
size_t
&
size
)
{
MACE_CHECK
(
data
!=
nullptr
&&
size
>
0
,
"data is null or size is 0"
);
int
ret
=
munmap
(
const_cast
<
unsigned
char
*>
(
data
),
size
);
MACE_CHECK
(
ret
==
0
,
"Failed to unmap file, error code: "
,
strerror
(
errno
));
}
}
// namespace mace
mace/utils/utils.h
浏览文件 @
3beb766d
...
...
@@ -15,12 +15,9 @@
#ifndef MACE_UTILS_UTILS_H_
#define MACE_UTILS_UTILS_H_
#include <
fstream
>
#include <
cstdlib
>
#include <map>
#include <sstream>
#include <string>
#include <cstdlib>
#include <utility>
#include <vector>
namespace
mace
{
...
...
@@ -70,51 +67,12 @@ Integer CeilQuotient(Integer a, Integer b) {
return
(
a
+
b
-
1
)
/
b
;
}
inline
std
::
string
ObfuscateString
(
const
std
::
string
&
src
,
const
std
::
string
&
lookup_table
)
{
std
::
string
dest
;
dest
.
resize
(
src
.
size
());
for
(
size_t
i
=
0
;
i
<
src
.
size
();
i
++
)
{
dest
[
i
]
=
src
[
i
]
^
lookup_table
[
i
%
lookup_table
.
size
()];
}
return
dest
;
}
std
::
string
ObfuscateString
(
const
std
::
string
&
src
,
const
std
::
string
&
lookup_table
);
// ObfuscateString(ObfuscateString(str)) ==> str
inline
std
::
string
ObfuscateString
(
const
std
::
string
&
src
)
{
// Keep consistent with obfuscation in python tools
return
ObfuscateString
(
src
,
"Mobile-AI-Compute-Engine"
);
}
std
::
string
ObfuscateString
(
const
std
::
string
&
src
);
// Obfuscate synbol or path string
inline
std
::
string
ObfuscateSymbol
(
const
std
::
string
&
src
)
{
std
::
string
dest
=
src
;
if
(
dest
.
empty
())
{
return
dest
;
}
dest
[
0
]
=
src
[
0
];
// avoid invalid symbol which starts from 0-9
const
std
::
string
encode_dict
=
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"
;
for
(
size_t
i
=
1
;
i
<
src
.
size
();
i
++
)
{
char
ch
=
src
[
i
];
int
idx
;
if
(
ch
>=
'0'
&&
ch
<=
'9'
)
{
idx
=
ch
-
'0'
;
}
else
if
(
ch
>=
'a'
&&
ch
<=
'z'
)
{
idx
=
10
+
ch
-
'a'
;
}
else
if
(
ch
>=
'A'
&&
ch
<=
'Z'
)
{
idx
=
10
+
26
+
ch
-
'a'
;
}
else
if
(
ch
==
'_'
)
{
idx
=
10
+
26
+
26
;
}
else
{
dest
[
i
]
=
ch
;
continue
;
}
// There is no collision if it's true for every char at every position
dest
[
i
]
=
encode_dict
[(
idx
+
i
+
31
)
%
encode_dict
.
size
()];
}
return
dest
;
}
std
::
string
ObfuscateSymbol
(
const
std
::
string
&
src
);
#ifdef MACE_OBFUSCATE_LITERALS
#define MACE_OBFUSCATE_STRING(str) ObfuscateString(str)
...
...
@@ -124,41 +82,17 @@ inline std::string ObfuscateSymbol(const std::string &src) {
#define MACE_OBFUSCATE_SYMBOL(str) (str)
#endif
inline
std
::
vector
<
std
::
string
>
Split
(
const
std
::
string
&
str
,
char
delims
)
{
std
::
vector
<
std
::
string
>
result
;
std
::
string
tmp
=
str
;
while
(
!
tmp
.
empty
())
{
size_t
next_offset
=
tmp
.
find
(
delims
);
result
.
push_back
(
tmp
.
substr
(
0
,
next_offset
));
if
(
next_offset
==
std
::
string
::
npos
)
{
break
;
}
else
{
tmp
=
tmp
.
substr
(
next_offset
+
1
);
}
}
return
result
;
}
std
::
vector
<
std
::
string
>
Split
(
const
std
::
string
&
str
,
char
delims
);
inline
bool
ReadBinaryFile
(
std
::
vector
<
unsigned
char
>
*
data
,
const
std
::
string
&
filename
)
{
std
::
ifstream
ifs
(
filename
,
std
::
ios
::
in
|
std
::
ios
::
binary
);
if
(
!
ifs
.
is_open
())
{
return
false
;
}
ifs
.
seekg
(
0
,
ifs
.
end
);
size_t
length
=
ifs
.
tellg
();
ifs
.
seekg
(
0
,
ifs
.
beg
);
data
->
reserve
(
length
);
data
->
insert
(
data
->
begin
(),
std
::
istreambuf_iterator
<
char
>
(
ifs
),
std
::
istreambuf_iterator
<
char
>
());
if
(
ifs
.
fail
())
{
return
false
;
}
ifs
.
close
();
bool
ReadBinaryFile
(
std
::
vector
<
unsigned
char
>
*
data
,
const
std
::
string
&
filename
);
return
true
;
}
void
MemoryMap
(
const
std
::
string
&
file
,
const
unsigned
char
**
data
,
size_t
*
size
);
void
MemoryUnMap
(
const
unsigned
char
*
data
,
const
size_t
&
size
);
template
<
typename
T
>
std
::
vector
<
std
::
string
>
MapKeys
(
const
std
::
map
<
std
::
string
,
T
>
&
data
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录