diff --git a/.gitignore b/.gitignore index 769047a4de7baaf6c9682196b76f08f9ba631ab0..894e0a843ad7e71d30cce504be890cf4755dea4c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ tags cmake-build-debug/ *.pyc mace/codegen/models/ +mace/codegen/opencl/ mace/codegen/opencl_bin/ mace/codegen/version/ diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 2d6a1b9bbae188fe8912f9018bee3683d5d315f1..0000000000000000000000000000000000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,107 +0,0 @@ -FROM ubuntu:16.04 - -# Update source -# Looks like mirrors.163.com does not work in xiaomi network -# RUN sed -i 's/http:\/\/archive\.ubuntu\.com\/ubuntu\//http:\/\/mirrors\.163\.com\/ubuntu\//g' /etc/apt/sources.list -RUN apt-get update -y - -## Basic tools -RUN apt-get install -y --no-install-recommends apt-utils -RUN apt-get install -y --no-install-recommends \ - build-essential \ - cmake \ - curl \ - git \ - libcurl3-dev \ - libgoogle-glog-dev \ - libfreetype6-dev \ - libpng12-dev \ - libprotobuf-dev \ - libzmq3-dev \ - pkg-config \ - python-dev \ - python-pip \ - protobuf-compiler \ - rsync \ - software-properties-common \ - unzip \ - zip \ - zlib1g-dev \ - openjdk-8-jdk \ - openjdk-8-jre-headless \ - wget - -ENV ANDROID_NDK_HOME /opt/android-ndk -ENV ANDROID_NDK /opt/android-ndk -ENV ANDROID_NDK_VERSION r12c - -# Android NDK -# download -RUN mkdir /opt/android-ndk-tmp && \ - cd /opt/android-ndk-tmp && \ - wget -q https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip && \ -# uncompress - unzip -q android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip && \ -# move to its final location - mv ./android-ndk-${ANDROID_NDK_VERSION} ${ANDROID_NDK_HOME} && \ -# remove temp dir - cd ${ANDROID_NDK_HOME} && \ - rm -rf /opt/android-ndk-tmp - -# add to PATH -ENV PATH ${PATH}:${ANDROID_NDK_HOME} - -# Set up Bazel. -# Running bazel inside a `docker build` command causes trouble, cf: -# https://github.com/bazelbuild/bazel/issues/134 -# The easiest solution is to set up a bazelrc file forcing --batch. -RUN echo "startup --batch" >>/etc/bazel.bazelrc -# Similarly, we need to workaround sandboxing issues: -# https://github.com/bazelbuild/bazel/issues/418 -RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ - >>/etc/bazel.bazelrc -# Install the most recent bazel release. -ENV BAZEL_VERSION 0.5.4 -WORKDIR / -RUN mkdir /bazel && \ - cd /bazel && \ - curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ - curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ - chmod +x bazel-*.sh && \ - ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ - cd / && \ - rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh - -# Setup vim -RUN apt-get update -y -RUN apt-get install -y --no-install-recommends \ - locales \ - vim - -RUN mkdir -p ~/.vim/autoload ~/.vim/bundle && \ - curl -LSso ~/.vim/autoload/pathogen.vim https://tpo.pe/pathogen.vim - -RUN mkdir -p ~/.vim/bundle -RUN cd ~/.vim/bundle && \ - git clone https://github.com/scrooloose/nerdtree.git && \ - git clone https://github.com/vim-syntastic/syntastic.git && \ - git clone https://github.com/vim-airline/vim-airline.git && \ - git clone https://github.com/altercation/vim-colors-solarized.git && \ - git clone https://github.com/bazelbuild/vim-ft-bzl.git && \ - git clone https://github.com/google/vim-maktaba.git && \ - git clone https://github.com/google/vim-codefmt.git - -RUN curl -LSso ~/.vimrc https://gist.githubusercontent.com/llhe/71c5802919debd5825c100c0135478a7/raw/16a35020cc65f9c25d0cf8f11a3ba7b345a1271d/.vimrc - -RUN locale-gen en_US.UTF-8 -ENV LC_CTYPE=en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -ENV TERM xterm-256color - -# Extra packges -RUN apt-get install -y --no-install-recommends \ - telnet \ - net-tools \ - inetutils-ping \ - screen \ - android-tools-adb diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 0f2719bb05fed231974bb39903ea2f64d97f4aa4..0000000000000000000000000000000000000000 --- a/docker/README.md +++ /dev/null @@ -1,27 +0,0 @@ -Development Docker Image -======== - -* Login in Xiaomi Docker Hub - - ``` - docker login docker.d.xiaomi.net - ``` - -* Build the image - - ``` - docker build -t docker.d.xiaomi.net/mace/mace-dev . - ``` - -* Push the image to Xiaomi Docker Hub - - ``` - docker push docker.d.xiaomi.net/mace/mace-dev - ``` - -* Pull and run the image - - ``` - # Set 'host' network to use ADB - docker run -it --net=host docker.d.xiaomi.net/mace/mace-dev /bin/bash - ``` diff --git a/tools/validate.py b/tools/validate.py deleted file mode 100644 index 9a7b890159af5497d721ae749cc2927d1154f431..0000000000000000000000000000000000000000 --- a/tools/validate.py +++ /dev/null @@ -1,133 +0,0 @@ -import argparse -import sys -import os -import os.path -import tensorflow as tf -import numpy as np -from scipy import spatial - -from tensorflow import gfile - -# Validation Flow: -# 1. Generate input data -# python validate_icnet.py --generate_data 1 -# -# 2. Use mace_run to run icnet on phone. -# 3. adb pull the result. -# 4. Compare output data of mace and tf -# python validate_icnet.py --model_file opt_icnet.pb \ -# --input_file input_file \ -# --mace_out_file icnet.out - -def generate_data(shape): - np.random.seed() - data = np.random.random(shape) * 2 - 1 - print FLAGS.input_file - data.astype(np.float32).tofile(FLAGS.input_file) - print "Generate input file done." - -def load_data(file): - if os.path.isfile(file): - return np.fromfile(file=file, dtype=np.float32) - else: - return np.empty([0]) - -def valid_output(out_shape, mace_out_file, tf_out_value): - mace_out_value = load_data(mace_out_file) - if mace_out_value.size != 0: - similarity = (1 - spatial.distance.cosine(tf_out_value.flat, mace_out_value)) - print 'MACE VS TF similarity: ', similarity - if similarity > 0.995: - print '=======================Similarity Test Passed======================' - else: - print '=======================Similarity Test Failed======================' - else: - print '=======================Skip empty node===================' - - -def run_model(input_shape): - if not gfile.Exists(FLAGS.model_file): - print("Input graph file '" + FLAGS.model_file + "' does not exist!") - return -1 - - input_graph_def = tf.GraphDef() - with gfile.Open(FLAGS.model_file, "rb") as f: - data = f.read() - input_graph_def.ParseFromString(data) - tf.import_graph_def(input_graph_def, name="") - - with tf.Session() as session: - with session.graph.as_default() as graph: - tf.import_graph_def(input_graph_def, name="") - input_node = graph.get_tensor_by_name(FLAGS.input_node + ':0') - output_node = graph.get_tensor_by_name(FLAGS.output_node + ':0') - - input_value = load_data(FLAGS.input_file) - input_value = input_value.reshape(input_shape) - - output_value = session.run(output_node, feed_dict={input_node: [input_value]}) - output_value.astype(np.float32).tofile( os.path.dirname(FLAGS.input_file) + '/tf_out') - return output_value - -def main(unused_args): - input_shape = [int(x) for x in FLAGS.input_shape.split(',')] - output_shape = [int(x) for x in FLAGS.output_shape.split(',')] - if FLAGS.generate_data: - generate_data(input_shape) - else: - output_value = run_model(input_shape) - valid_output(output_shape, FLAGS.mace_out_file, output_value) - - -def parse_args(): - """Parses command line arguments.""" - parser = argparse.ArgumentParser() - parser.register("type", "bool", lambda v: v.lower() == "true") - parser.add_argument( - "--model_file", - type=str, - default="", - help="TensorFlow \'GraphDef\' file to load.") - parser.add_argument( - "--input_file", - type=str, - default="", - help="input file.") - parser.add_argument( - "--mace_out_file", - type=str, - default="", - help="mace output file to load.") - parser.add_argument( - "--input_shape", - type=str, - default="512,512,3", - help="input shape.") - parser.add_argument( - "--output_shape", - type=str, - default="1,512,512,2", - help="output shape.") - parser.add_argument( - "--input_node", - type=str, - default="input_node", - help="input node") - parser.add_argument( - "--output_node", - type=str, - default="output_node", - help="output node") - parser.add_argument( - "--generate_data", - type='bool', - default="false", - help="Generate data or not.") - - return parser.parse_known_args() - - -if __name__ == '__main__': - FLAGS, unparsed = parse_args() - main(unused_args=[sys.argv[0]] + unparsed) - diff --git a/tools/validate_icnet.sh b/tools/validate_icnet.sh deleted file mode 100755 index b7c0bb6e66d8f4212d12467b7e4276e5ac28e9f7..0000000000000000000000000000000000000000 --- a/tools/validate_icnet.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -# Must run at root dir of mace project. -set -e - -Usage() { - echo 'Usage: bash tools/validate_icnet.sh tf_model_file' -} - -if [ $# != 1 ];then - Usage - exit -1 -fi - -TF_MODEL_FILE_PATH=$1 -MODEL_DIR=$(dirname ${TF_MODEL_FILE_PATH}) -MACE_MODEL_NAME='mace_model.pb' -TF_INPUT_FILE_NAME='tf_model_input' -MACE_INPUT_FILE_NAME='mace_model_input' -OUTPUT_FILE_NAME='icnet.out' -PHONE_DATA_DIR="/data/local/tmp/${MACE_MODEL_NAME}" -KERNEL_DIR="${PHONE_DATA_DIR}/cl/" - -# Step 1: convert tf model to mace model -echo "Step 1: convert tf model to mace model" -bazel build //mace/python/tools:tf_converter -bazel-bin/mace/python/tools/tf_converter --input=${TF_MODEL_FILE_PATH} --output=${MODEL_DIR}/${MACE_MODEL_NAME} - -# Step 2: Generate input data -echo "Step 2: Generate input data" -python tools/validate_icnet.py --generate_data true --random_seed 1 \ - --mace_input_file ${MODEL_DIR}/${MACE_INPUT_FILE_NAME} --tf_input_file ${MODEL_DIR}/${TF_INPUT_FILE_NAME} - -# Step 3: Run model on the phone -echo "Step 3: Run model on the phone" -bazel build -c opt --strip always mace/examples:mace_run \ - --crosstool_top=//external:android/crosstool \ - --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ - --cpu=arm64-v8a - -adb shell "mkdir -p ${PHONE_DATA_DIR}" -adb shell "mkdir -p ${KERNEL_DIR}" -adb push mace/kernels/opencl/cl/* ${KERNEL_DIR} -adb push ${MODEL_DIR}/${MACE_MODEL_NAME} ${PHONE_DATA_DIR} -adb push ${MODEL_DIR}/${MACE_INPUT_FILE_NAME} ${PHONE_DATA_DIR} -adb push bazel-bin/mace/examples/mace_run ${PHONE_DATA_DIR} - -num_threads=${1:-1} - -adb shell MACE_RUN_PARAMETER_PATH=${PHONE_DATA_DIR}/mace_run.config \ - MACE_KERNEL_PATH=$KERNEL_DIR \ - OMP_NUM_THREADS=$num_threads \ - ${PHONE_DATA_DIR}/mace_run \ - --input_shape=1,3,480,480\ - --input_file=${PHONE_DATA_DIR}/${MACE_INPUT_FILE_NAME} \ - --output_file=${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} \ - --device=OPENCL - -# Step 4: pull the mace run result. -echo "Step 4: pull the mace run result." -adb pull ${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} ${MODEL_DIR} - -# Step 5: validate the result -echo "Step 5: validate the result" -python tools/validate_icnet.py --model_file ${TF_MODEL_FILE_PATH} \ - --tf_input_file ${MODEL_DIR}/${TF_INPUT_FILE_NAME} \ - --mace_out_file ${MODEL_DIR}/${OUTPUT_FILE_NAME} - -