From b7145c73cbd2f68f0bff840a1972a08bbbbe21e6 Mon Sep 17 00:00:00 2001 From: Liangliang He Date: Fri, 23 Mar 2018 14:05:20 +0800 Subject: [PATCH] Misc code improvements --- docker/Dockerfile | 3 +- mace/python/tools/binary_codegen.py | 2 +- mace/python/tools/encrypt_opencl_codegen.py | 2 +- .../tools/{model.template => model.jinja2} | 27 ++++++++--------- ...el_header.template => model_header.jinja2} | 0 mace/python/tools/opencl_codegen.py | 2 +- .../{operator.template => operator.jinja2} | 28 ++++++++--------- mace/python/tools/source_converter_lib.py | 10 +++---- ...ec_maps.cc.tmpl => str2vec_maps.cc.jinja2} | 0 ...ensor_data.template => tensor_data.jinja2} | 0 ...r_source.template => tensor_source.jinja2} | 4 +-- tools/adb_tools.py | 30 +++++++++++++++++++ tools/mace_tools.py | 23 +++++++++++++- 13 files changed, 91 insertions(+), 40 deletions(-) rename mace/python/tools/{model.template => model.jinja2} (84%) rename mace/python/tools/{model_header.template => model_header.jinja2} (100%) rename mace/python/tools/{operator.template => operator.jinja2} (85%) rename mace/python/tools/{str2vec_maps.cc.tmpl => str2vec_maps.cc.jinja2} (100%) rename mace/python/tools/{tensor_data.template => tensor_data.jinja2} (100%) rename mace/python/tools/{tensor_source.template => tensor_source.jinja2} (92%) create mode 100644 tools/adb_tools.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 6aa2813c..94c19f12 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -112,7 +112,8 @@ RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com RUN pip install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com tensorflow==1.4.0 \ scipy \ jinja2 \ - pyyaml + pyyaml \ + sh # Download tensorflow tools RUN wget http://cnbj1-inner-fds.api.xiaomi.net/mace/tool/transform_graph && \ diff --git a/mace/python/tools/binary_codegen.py b/mace/python/tools/binary_codegen.py index aea06a0a..9956624b 100644 --- a/mace/python/tools/binary_codegen.py +++ b/mace/python/tools/binary_codegen.py @@ -44,7 +44,7 @@ def generate_cpp_source(): idx += params_size env = jinja2.Environment(loader=jinja2.FileSystemLoader(sys.path[0])) - return env.get_template('str2vec_maps.cc.tmpl').render( + return env.get_template('str2vec_maps.cc.jinja2').render( maps = data_map, data_type = 'unsigned int', variable_name = FLAGS.variable_name diff --git a/mace/python/tools/encrypt_opencl_codegen.py b/mace/python/tools/encrypt_opencl_codegen.py index 04a4cfc3..ad3ab8d4 100644 --- a/mace/python/tools/encrypt_opencl_codegen.py +++ b/mace/python/tools/encrypt_opencl_codegen.py @@ -45,7 +45,7 @@ def main(unused_args): encrypted_code_maps[file_name[:-3]] = encrypted_code_arr env = jinja2.Environment(loader=jinja2.FileSystemLoader(sys.path[0])) - cpp_cl_encrypted_kernel = env.get_template('str2vec_maps.cc.tmpl').render( + cpp_cl_encrypted_kernel = env.get_template('str2vec_maps.cc.jinja2').render( maps=encrypted_code_maps, data_type='unsigned char', variable_name='kEncryptedProgramMap') diff --git a/mace/python/tools/model.template b/mace/python/tools/model.jinja2 similarity index 84% rename from mace/python/tools/model.template rename to mace/python/tools/model.jinja2 index 8ad245ef..4baf86fe 100644 --- a/mace/python/tools/model.template +++ b/mace/python/tools/model.jinja2 @@ -14,13 +14,13 @@ namespace mace { namespace {{tag}} { {% for tensor in tensors %} -extern void CreateTensor{{ tensor.id }}(std::vector &tensors, +extern void CreateTensor{{ tensor.id }}(std::vector *tensors, const unsigned char *model_data); {% endfor %} {% for i in range(net.op|length) %} -extern void CreateOperator{{i}}(mace::OperatorDef &op); +extern void CreateOperator{{i}}(mace::OperatorDef *op); {% endfor %} } // namespace {{ tag }} @@ -79,31 +79,30 @@ void CreateOutputInfo(mace::NetDef &net_def) { } {% endif %} -void CreateOperators(std::vector &ops) { +void CreateOperators(std::vector *ops) { MACE_LATENCY_LOGGER(1, "Create operators"); - ops.resize({{ net.op|length }}); - {% for i in range(net.op|length) %} + ops->resize({{ net.op|length }}); - mace::{{tag}}::CreateOperator{{i}}(ops[{{i}}]); + {% for i in range(net.op|length) %} + mace::{{tag}}::CreateOperator{{i}}(&ops->at({{i}})); {% endfor %} } -void CreateTensors(std::vector &tensors, +void CreateTensors(std::vector *tensors, const unsigned char *model_data) { MACE_LATENCY_LOGGER(1, "Create tensors"); - tensors.reserve({{ net.tensors|length }}); + tensors->reserve({{ net.tensors|length }}); {% for tensor in tensors %} - mace::{{tag}}::CreateTensor{{tensor.id}}(tensors, model_data); {% endfor %} } {% if net.mem_arena.mem_block|length != 0 %} -void CreateMemoryArena(mace::MemoryArena &mem_arena) { - std::vector &mem_block = mem_arena.mutable_mem_block(); +void CreateMemoryArena(mace::MemoryArena *mem_arena) { + std::vector &mem_block = mem_arena->mutable_mem_block(); mem_block.reserve({{ net.mem_arena.mem_block|length }}); {% for mem_blk in net.mem_arena.mem_block %} @@ -129,12 +128,12 @@ NetDef CreateNet(const unsigned char *model_data) { CreateNetArg(net_def); {% endif %} - CreateOperators(net_def.mutable_op()); + CreateOperators(&net_def.mutable_op()); - CreateTensors(net_def.mutable_tensors(), model_data); + CreateTensors(&net_def.mutable_tensors(), model_data); {% if net.mem_arena.mem_block|length != 0 %} - CreateMemoryArena(net_def.mutable_mem_arena()); + CreateMemoryArena(&net_def.mutable_mem_arena()); {% endif %} {% if net.output_info | length > 0 %} diff --git a/mace/python/tools/model_header.template b/mace/python/tools/model_header.jinja2 similarity index 100% rename from mace/python/tools/model_header.template rename to mace/python/tools/model_header.jinja2 diff --git a/mace/python/tools/opencl_codegen.py b/mace/python/tools/opencl_codegen.py index d5109326..04bda9a8 100644 --- a/mace/python/tools/opencl_codegen.py +++ b/mace/python/tools/opencl_codegen.py @@ -31,7 +31,7 @@ def generate_cpp_source(): maps[file_name[:-4]].append(hex(ele)) env = jinja2.Environment(loader=jinja2.FileSystemLoader(sys.path[0])) - return env.get_template('str2vec_maps.cc.tmpl').render( + return env.get_template('str2vec_maps.cc.jinja2').render( maps = maps, data_type = 'unsigned char', variable_name = 'kCompiledProgramMap' diff --git a/mace/python/tools/operator.template b/mace/python/tools/operator.jinja2 similarity index 85% rename from mace/python/tools/operator.template rename to mace/python/tools/operator.jinja2 index bdbcbbcb..7c142ca9 100644 --- a/mace/python/tools/operator.template +++ b/mace/python/tools/operator.jinja2 @@ -13,7 +13,7 @@ namespace mace { namespace { -void UpdateOp(mace::OperatorDef &op, +void UpdateOp(mace::OperatorDef *op, const std::string &name, const std::string &type, const std::vector &inputs, @@ -21,13 +21,13 @@ void UpdateOp(mace::OperatorDef &op, const std::vector &output_types, uint32_t node_id, const std::vector &mem_ids) { - op.set_name(name); - op.set_type(type); - op.set_input(inputs); - op.set_output(outputs); - op.set_output_type(output_types); - op.set_node_id(node_id); - op.set_mem_id(mem_ids); + op->set_name(name); + op->set_type(type); + op->set_input(inputs); + op->set_output(outputs); + op->set_output_type(output_types); + op->set_node_id(node_id); + op->set_mem_id(mem_ids); } } // namespace @@ -38,13 +38,13 @@ namespace {{tag}} { {% for i in range(start, end) %} -void CreateOperator{{i}}(mace::OperatorDef &op) { +void CreateOperator{{i}}(mace::OperatorDef *op) { MACE_LATENCY_LOGGER(2, "Create operator {{ net.op[i].name }}"); mace::Argument *arg = nullptr; {% for arg in net.op[i].arg %} - arg = op.add_arg(); + arg = op->add_arg(); arg->set_name({{ arg.name|tojson }}); {%- if arg.HasField('f') %} @@ -70,7 +70,7 @@ void CreateOperator{{i}}(mace::OperatorDef &op) { {% for shape in net.op[i].output_shape %} {% if shape.dims | length > 0 %} - op.add_output_shape(mace::OutputShape({ {{ shape.dims|join(', ') }} })); + op->add_output_shape(mace::OutputShape({ {{ shape.dims|join(', ') }} })); {% endif %} {% endfor %} @@ -87,20 +87,20 @@ void CreateOperator{{i}}(mace::OperatorDef &op) { { {{ net.op[i].mem_id | join(', ') }} }); {% if runtime == 'dsp' %} - op.set_padding({{ net.op[i].padding }}); + op->set_padding({{ net.op[i].padding }}); {% if net.op[i].node_input | length > 0 %} std::vector input_node_ids({ {{ net.op[i].node_input | map(attribute='node_id') | join(', ') }} }); std::vector input_output_ports({ {{ net.op[i].node_input | map(attribute='output_port') | join(', ')}} }); for (size_t i = 0; i < {{ net.op[i].node_input | length }}; ++i) { mace::NodeInput input(input_node_ids[i], input_output_ports[i]); - op.add_node_input(input); + op->add_node_input(input); } {% endif %} {% if net.op[i].out_max_byte_size | length > 0 %} std::vector out_max_byte_sizes {{ net.op[i].out_max_byte_size | replace('[', '{') | replace(']', '}') }}; for (size_t i = 0; i < {{ net.op[i].out_max_byte_size | length }}; ++i) { - op.add_out_max_byte_size(out_max_byte_sizes[i]); + op->add_out_max_byte_size(out_max_byte_sizes[i]); } {% endif %} {% endif %} diff --git a/mace/python/tools/source_converter_lib.py b/mace/python/tools/source_converter_lib.py index 52fc6126..16ad18d4 100644 --- a/mace/python/tools/source_converter_lib.py +++ b/mace/python/tools/source_converter_lib.py @@ -110,7 +110,7 @@ def convert_to_source(net_def, mode_pb_checksum, template_dir, obfuscate, model_ j2_env.filters['stringfy'] = stringfy output_dir = os.path.dirname(output) + '/' # generate tensor source files - template_name = 'tensor_source.template' + template_name = 'tensor_source.jinja2' model_data = [] offset = 0 counter = 0 @@ -135,7 +135,7 @@ def convert_to_source(net_def, mode_pb_checksum, template_dir, obfuscate, model_ counter += 1 # generate tensor data - template_name = 'tensor_data.template' + template_name = 'tensor_data.jinja2' source = j2_env.get_template(template_name).render( tag = model_tag, embed_model_data = embed_model_data, @@ -150,7 +150,7 @@ def convert_to_source(net_def, mode_pb_checksum, template_dir, obfuscate, model_ f.close() # generate op source files - template_name = 'operator.template' + template_name = 'operator.jinja2' counter = 0 op_size = len(net_def.op) for start in range(0, op_size, 10): @@ -166,7 +166,7 @@ def convert_to_source(net_def, mode_pb_checksum, template_dir, obfuscate, model_ counter += 1 # generate model source files - template_name = 'model.template' + template_name = 'model.jinja2' tensors = [TensorInfo(i, net_def.tensors[i], runtime) for i in range(len(net_def.tensors))] source = j2_env.get_template(template_name).render( tensors = tensors, @@ -179,7 +179,7 @@ def convert_to_source(net_def, mode_pb_checksum, template_dir, obfuscate, model_ f.write(source) # generate model header file - template_name = 'model_header.template' + template_name = 'model_header.jinja2' source = j2_env.get_template(template_name).render( tag = model_tag, ) diff --git a/mace/python/tools/str2vec_maps.cc.tmpl b/mace/python/tools/str2vec_maps.cc.jinja2 similarity index 100% rename from mace/python/tools/str2vec_maps.cc.tmpl rename to mace/python/tools/str2vec_maps.cc.jinja2 diff --git a/mace/python/tools/tensor_data.template b/mace/python/tools/tensor_data.jinja2 similarity index 100% rename from mace/python/tools/tensor_data.template rename to mace/python/tools/tensor_data.jinja2 diff --git a/mace/python/tools/tensor_source.template b/mace/python/tools/tensor_source.jinja2 similarity index 92% rename from mace/python/tools/tensor_source.template rename to mace/python/tools/tensor_source.jinja2 index c321112e..f98482a3 100644 --- a/mace/python/tools/tensor_source.template +++ b/mace/python/tools/tensor_source.jinja2 @@ -13,10 +13,10 @@ namespace mace { namespace {{tag}} { -void CreateTensor{{tensor_info.id}}(std::vector &tensors, +void CreateTensor{{tensor_info.id}}(std::vector *tensors, const unsigned char *model_data) { MACE_LATENCY_LOGGER(2, "Create tensor {{ tensor.name }}"); - tensors.emplace_back(mace::ConstTensor( + tensors->emplace_back(mace::ConstTensor( {{ tensor.name|tojson }}, model_data + {{ offset }}, { {{ tensor.dims|join(', ') }} }, {{ tensor_info.data_type }}, {{ tensor.node_id }})); } diff --git a/tools/adb_tools.py b/tools/adb_tools.py new file mode 100644 index 00000000..8dfd92bd --- /dev/null +++ b/tools/adb_tools.py @@ -0,0 +1,30 @@ +import sh +import re + +def adb_split_stdout(stdout_str): + # Filter out last empty line + return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0] + +def adb_devices(): + outputs = sh.grep(sh.adb("devices"), "^[A-Za-z0-9]\+[[:space:]]\+device$") + raw_lists = sh.cut(outputs, "-f1") + return adb_split_stdout(raw_lists) + +def adb_getprop_by_serialno(serialno): + outputs = sh.adb("-s", serialno, "shell", "getprop") + raw_props = adb_split_stdout(outputs) + + props = {} + p = re.compile("\[(.+)\]: \[(.+)\]") + for raw_prop in raw_props: + m = p.match(raw_prop) + if m: + props[m.group(1)] = m.group(2) + return props + +def adb_get_all_socs(): + socs = [] + for d in adb_devices(): + props = adb_getprop_by_serialno(d) + socs.append(props["ro.product.board"]) + return set(socs) diff --git a/tools/mace_tools.py b/tools/mace_tools.py index a9aebdc9..78c42fa1 100644 --- a/tools/mace_tools.py +++ b/tools/mace_tools.py @@ -15,6 +15,8 @@ import sys import urllib import yaml +import adb_tools + from ConfigParser import ConfigParser @@ -201,6 +203,11 @@ def parse_args(): type=str, default="all", help="[build|run|validate|merge|all|throughput_test].") + parser.add_argument( + "--socs", + type=str, + default="all", + help="SoCs to build, comma seperated list (getprop ro.board.platform)") return parser.parse_known_args() @@ -227,7 +234,21 @@ def main(unused_args): generate_opencl_and_version_code() option_args = ' '.join([arg for arg in unused_args if arg.startswith('--')]) - for target_soc in configs["target_socs"]: + available_socs = adb_tools.adb_get_all_socs() + target_socs = available_socs + if hasattr(configs, "target_socs"): + target_socs = set(configs["target_socs"]) + target_socs = target_socs & available_socs + + if FLAGS.socs != "all": + socs = set(FLAGS.socs.split(',')) + target_socs = target_socs & socs + missing_socs = socs.difference(target_socs) + if len(missing_socs) > 0: + print("Error: devices with SoCs are not connected %s" % missing_socs) + exit(1) + + for target_soc in target_socs: for target_abi in configs["target_abis"]: global_runtime = get_global_runtime(configs) # Transfer params by environment -- GitLab