Merge branch 'move_dockerfile_to_libmace_and_remove_unused_scripts' into 'master'

move dockerfile to libmace and remove unused scripts

See merge request !221

.gitignore

@@ -4,5 +4,6 @@ tags
 cmake-build-debug/
 *.pyc
 mace/codegen/models/
+mace/codegen/opencl/
 mace/codegen/opencl_bin/
 mace/codegen/version/

docker/Dockerfile

-FROM ubuntu:16.04
-
-# Update source
-# Looks like mirrors.163.com does not work in xiaomi network
-# RUN sed -i 's/http:\/\/archive\.ubuntu\.com\/ubuntu\//http:\/\/mirrors\.163\.com\/ubuntu\//g' /etc/apt/sources.list
-RUN apt-get update -y
-
-## Basic tools
-RUN apt-get install -y --no-install-recommends apt-utils
-RUN apt-get install -y --no-install-recommends \
-    build-essential \
-    cmake \
-    curl \
-    git \
-    libcurl3-dev \
-    libgoogle-glog-dev \
-    libfreetype6-dev \
-    libpng12-dev \
-    libprotobuf-dev \
-    libzmq3-dev \
-    pkg-config \
-    python-dev \
-    python-pip \
-    protobuf-compiler \
-    rsync \
-    software-properties-common \
-    unzip \
-    zip \
-    zlib1g-dev \
-    openjdk-8-jdk \
-    openjdk-8-jre-headless \
-    wget
-
-ENV ANDROID_NDK_HOME /opt/android-ndk
-ENV ANDROID_NDK /opt/android-ndk
-ENV ANDROID_NDK_VERSION r12c
-
-# Android NDK
-# download
-RUN mkdir /opt/android-ndk-tmp && \
-    cd /opt/android-ndk-tmp && \
-    wget -q https://dl.google.com/android/repository/android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip && \
-# uncompress
-    unzip -q android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip && \
-# move to its final location
-    mv ./android-ndk-${ANDROID_NDK_VERSION} ${ANDROID_NDK_HOME} && \
-# remove temp dir
-    cd ${ANDROID_NDK_HOME} && \
-    rm -rf /opt/android-ndk-tmp
-
-# add to PATH
-ENV PATH ${PATH}:${ANDROID_NDK_HOME}
-
-# Set up Bazel.
-# Running bazel inside a `docker build` command causes trouble, cf:
-#   https://github.com/bazelbuild/bazel/issues/134
-# The easiest solution is to set up a bazelrc file forcing --batch.
-RUN echo "startup --batch" >>/etc/bazel.bazelrc
-# Similarly, we need to workaround sandboxing issues:
-#   https://github.com/bazelbuild/bazel/issues/418
-RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
-    >>/etc/bazel.bazelrc
-# Install the most recent bazel release.
-ENV BAZEL_VERSION 0.5.4
-WORKDIR /
-RUN mkdir /bazel && \
-    cd /bazel && \
-    curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
-    curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \
-    chmod +x bazel-*.sh && \
-    ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
-    cd / && \
-    rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
-
-# Setup vim
-RUN apt-get update -y
-RUN apt-get install -y --no-install-recommends \
-    locales \
-    vim
-
-RUN mkdir -p ~/.vim/autoload ~/.vim/bundle && \
-    curl -LSso ~/.vim/autoload/pathogen.vim https://tpo.pe/pathogen.vim
-
-RUN mkdir -p ~/.vim/bundle
-RUN cd ~/.vim/bundle && \
-    git clone https://github.com/scrooloose/nerdtree.git && \
-    git clone https://github.com/vim-syntastic/syntastic.git && \
-    git clone https://github.com/vim-airline/vim-airline.git && \
-    git clone https://github.com/altercation/vim-colors-solarized.git && \
-    git clone https://github.com/bazelbuild/vim-ft-bzl.git && \
-    git clone https://github.com/google/vim-maktaba.git && \
-    git clone https://github.com/google/vim-codefmt.git
-
-RUN curl -LSso ~/.vimrc  https://gist.githubusercontent.com/llhe/71c5802919debd5825c100c0135478a7/raw/16a35020cc65f9c25d0cf8f11a3ba7b345a1271d/.vimrc
-
-RUN locale-gen en_US.UTF-8
-ENV LC_CTYPE=en_US.UTF-8
-ENV LC_ALL=en_US.UTF-8
-ENV TERM xterm-256color
-
-# Extra packges
-RUN apt-get install -y --no-install-recommends \
-    telnet \
-    net-tools \
-    inetutils-ping \
-    screen \
-    android-tools-adb

docker/README.md

-Development Docker Image
-========
-
-* Login in Xiaomi Docker Hub
-
-  ```
-  docker login docker.d.xiaomi.net
-  ```
-
-* Build the image
-
-  ```
-  docker build -t docker.d.xiaomi.net/mace/mace-dev .
-  ```
-
-* Push the image to Xiaomi Docker Hub
-
-  ```
-  docker push docker.d.xiaomi.net/mace/mace-dev
-  ```
-
-* Pull and run the image
-
-  ```
-  # Set 'host' network to use ADB
-  docker run -it --net=host docker.d.xiaomi.net/mace/mace-dev /bin/bash
-  ```

tools/validate.py

-import argparse
-import sys
-import os
-import os.path
-import tensorflow as tf
-import numpy as np
-from scipy import spatial
-
-from tensorflow import gfile
-
-# Validation Flow:
-# 1. Generate input data
-#    python validate_icnet.py --generate_data 1
-#
-# 2. Use mace_run to run icnet on phone.
-# 3. adb pull the result.
-# 4. Compare output data of mace and tf
-#    python validate_icnet.py --model_file opt_icnet.pb \
-#        --input_file input_file \
-#        --mace_out_file icnet.out
-
-def generate_data(shape):
-  np.random.seed()
-  data = np.random.random(shape) * 2 - 1
-  print FLAGS.input_file
-  data.astype(np.float32).tofile(FLAGS.input_file)
-  print "Generate input file done."
-
-def load_data(file):
-  if os.path.isfile(file):
-    return np.fromfile(file=file, dtype=np.float32)
-  else:
-    return np.empty([0])
-
-def valid_output(out_shape, mace_out_file, tf_out_value):
-  mace_out_value = load_data(mace_out_file)
-  if mace_out_value.size != 0:
-    similarity = (1 - spatial.distance.cosine(tf_out_value.flat, mace_out_value))
-    print 'MACE VS TF similarity: ', similarity
-    if similarity > 0.995:
-      print '=======================Similarity Test Passed======================'
-    else:
-      print '=======================Similarity Test Failed======================'
-  else:
-    print '=======================Skip empty node==================='
-
-
-def run_model(input_shape):
-  if not gfile.Exists(FLAGS.model_file):
-    print("Input graph file '" + FLAGS.model_file + "' does not exist!")
-    return -1
-
-  input_graph_def = tf.GraphDef()
-  with gfile.Open(FLAGS.model_file, "rb") as f:
-    data = f.read()
-    input_graph_def.ParseFromString(data)
-    tf.import_graph_def(input_graph_def, name="")
-
-    with tf.Session() as session:
-      with session.graph.as_default() as graph:
-        tf.import_graph_def(input_graph_def, name="")
-        input_node = graph.get_tensor_by_name(FLAGS.input_node + ':0')
-        output_node = graph.get_tensor_by_name(FLAGS.output_node + ':0')
-
-        input_value = load_data(FLAGS.input_file)
-        input_value = input_value.reshape(input_shape)
-        
-        output_value = session.run(output_node, feed_dict={input_node: [input_value]})
-        output_value.astype(np.float32).tofile( os.path.dirname(FLAGS.input_file) + '/tf_out')
-        return output_value
-
-def main(unused_args):
-  input_shape = [int(x) for x in FLAGS.input_shape.split(',')]
-  output_shape = [int(x) for x in FLAGS.output_shape.split(',')]
-  if FLAGS.generate_data:
-    generate_data(input_shape)
-  else:
-    output_value = run_model(input_shape)
-    valid_output(output_shape, FLAGS.mace_out_file, output_value)
-
-
-def parse_args():
-  """Parses command line arguments."""
-  parser = argparse.ArgumentParser()
-  parser.register("type", "bool", lambda v: v.lower() == "true")
-  parser.add_argument(
-    "--model_file",
-    type=str,
-    default="",
-    help="TensorFlow \'GraphDef\' file to load.")
-  parser.add_argument(
-    "--input_file",
-    type=str,
-    default="",
-    help="input file.")
-  parser.add_argument(
-    "--mace_out_file",
-    type=str,
-    default="",
-    help="mace output file to load.")
-  parser.add_argument(
-    "--input_shape",
-    type=str,
-    default="512,512,3",
-    help="input shape.")
-  parser.add_argument(
-    "--output_shape",
-    type=str,
-    default="1,512,512,2",
-    help="output shape.")
-  parser.add_argument(
-    "--input_node",
-    type=str,
-    default="input_node",
-    help="input node")
-  parser.add_argument(
-    "--output_node",
-    type=str,
-    default="output_node",
-    help="output node")
-  parser.add_argument(
-    "--generate_data",
-    type='bool',
-    default="false",
-    help="Generate data or not.")
-
-  return parser.parse_known_args()
-
-
-if __name__ == '__main__':
-  FLAGS, unparsed = parse_args()
-  main(unused_args=[sys.argv[0]] + unparsed)
-

tools/validate_icnet.sh

-#!/bin/bash
-# Must run at root dir of mace project.
-set -e
-
-Usage() {
-  echo 'Usage: bash tools/validate_icnet.sh tf_model_file'
-}
-
-if [ $# != 1 ];then
-  Usage
-  exit -1
-fi
-
-TF_MODEL_FILE_PATH=$1
-MODEL_DIR=$(dirname ${TF_MODEL_FILE_PATH})
-MACE_MODEL_NAME='mace_model.pb'
-TF_INPUT_FILE_NAME='tf_model_input'
-MACE_INPUT_FILE_NAME='mace_model_input'
-OUTPUT_FILE_NAME='icnet.out'
-PHONE_DATA_DIR="/data/local/tmp/${MACE_MODEL_NAME}"
-KERNEL_DIR="${PHONE_DATA_DIR}/cl/"
-
-# Step 1: convert tf model to mace model
-echo "Step 1: convert tf model to mace model"
-bazel build //mace/python/tools:tf_converter
-bazel-bin/mace/python/tools/tf_converter --input=${TF_MODEL_FILE_PATH} --output=${MODEL_DIR}/${MACE_MODEL_NAME}
-
-# Step 2: Generate input data
-echo "Step 2: Generate input data"
-python tools/validate_icnet.py --generate_data true --random_seed 1 \
- --mace_input_file ${MODEL_DIR}/${MACE_INPUT_FILE_NAME} --tf_input_file ${MODEL_DIR}/${TF_INPUT_FILE_NAME}
-
-# Step 3: Run model on the phone
-echo "Step 3: Run model on the phone"
-bazel build -c opt --strip always mace/examples:mace_run  \
-    --crosstool_top=//external:android/crosstool \
-    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
-    --cpu=arm64-v8a
-
-adb shell "mkdir -p ${PHONE_DATA_DIR}"
-adb shell "mkdir -p ${KERNEL_DIR}"
-adb push mace/kernels/opencl/cl/* ${KERNEL_DIR}
-adb push ${MODEL_DIR}/${MACE_MODEL_NAME} ${PHONE_DATA_DIR}
-adb push ${MODEL_DIR}/${MACE_INPUT_FILE_NAME} ${PHONE_DATA_DIR}
-adb push bazel-bin/mace/examples/mace_run ${PHONE_DATA_DIR}
-
-num_threads=${1:-1}
-
-adb shell MACE_RUN_PARAMETER_PATH=${PHONE_DATA_DIR}/mace_run.config \
-          MACE_KERNEL_PATH=$KERNEL_DIR \
-          OMP_NUM_THREADS=$num_threads \
-          ${PHONE_DATA_DIR}/mace_run \
-            --input_shape=1,3,480,480\
-            --input_file=${PHONE_DATA_DIR}/${MACE_INPUT_FILE_NAME} \
-            --output_file=${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} \
-            --device=OPENCL
-
-# Step 4: pull the mace run result.
-echo "Step 4: pull the mace run result."
-adb pull ${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} ${MODEL_DIR}
-
-# Step 5: validate the result
-echo "Step 5: validate the result"
-python tools/validate_icnet.py --model_file ${TF_MODEL_FILE_PATH} \
-  --tf_input_file ${MODEL_DIR}/${TF_INPUT_FILE_NAME} \
-  --mace_out_file ${MODEL_DIR}/${OUTPUT_FILE_NAME}
-
-