Add script to call webpack to deploy and watch frontend code from within...

Add script to call webpack to deploy and watch frontend code from within flask.  Optimize script location (#236)

* Add script to call webpack to deploy and watch frontend code from within flask.  Optimize script location

* remove npm install

* Create "setup_dev_env.sh" script that will set up all the node and SDK depenencies.  Create "start_dev_server.sh" to start up dev server, which will start backend server and start webpack --watch on frontend.

.travis.yml

@@ -41,8 +41,8 @@ before_install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install brew-pip; fi
 
 script:
-  - if [[ "$JOB" == "check_style" ]]; then ./travis/check_style.sh; fi
-  - if [[ "$JOB" == "test" ]]; then /bin/bash ./tests.sh all; fi
+  - if [[ "$JOB" == "check_style" ]]; then ./scripts/check_style.sh; fi
+  - if [[ "$JOB" == "test" ]]; then ./scripts/tests.sh all; fi
   - |
     if [[ "$JOB" != "build_doc" ]]; then exit 0; fi;
     if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then exit 0; fi;

frontend/tool/webpack.config.js

@@ -124,7 +124,8 @@ const config = {
         new CaseSensitivePathsPlugin(),
         new webpack.LoaderOptionsPlugin({
             test: /\.(styl|san)$/
-        })
+        }),
+        new ExtractTextPlugin({filename: '[name].css'})
     ]
 };
 

scripts/build.sh

+#!/bin/bash
+set -ex
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $SCRIPT_DIR/..
+
+TOP_DIR=$(pwd)
+FRONTEND_DIR=$TOP_DIR/frontend
+BACKEND_DIR=$TOP_DIR/visualdl
+BUILD_DIR=$TOP_DIR/build
+
+mkdir -p $BUILD_DIR
+
+check_duplicated() {
+    filename_format=$1
+    file_num=`ls dist/${filename_format} | wc -l | awk '{$1=$1;print}'`
+    if [ "$file_num" != "1" ]; then
+      echo "dist have duplicate file for $file_num, please clean and rerun"
+      exit 1
+    fi
+}
+
+build_frontend() {
+    cd $FRONTEND_DIR
+    if [ ! -d "dist" ]; then
+      npm install
+      npm run build
+    fi
+    for file_name in "manifest.*.js" "index.*.js" "vendor.*.js"; do
+        echo $file_name
+        check_duplicated $file_name
+    done
+}
+
+build_frontend_fake() {
+    cd $FRONTEND_DIR
+    mkdir -p dist
+}
+
+build_backend() {
+    cd $BUILD_DIR
+    cmake .. ${PYTHON_FLAGS}
+    make -j2
+}
+
+build_onnx_graph() {
+    export PATH="$BUILD_DIR/third_party/protobuf/src/extern_protobuf-build/:$PATH"
+    cd $TOP_DIR/visualdl/server/onnx
+    protoc onnx.proto --python_out .
+}
+
+clean_env() {
+    rm -rf $TOP_DIR/visualdl/server/dist
+    rm -rf $BUILD_DIR/bdist*
+    rm -rf $BUILD_DIR/lib*
+    rm -rf $BUILD_DIR/temp*
+    rm -rf $BUILD_DIR/scripts*
+}
+
+package() {
+    cp -rf $FRONTEND_DIR/dist $TOP_DIR/visualdl/server/
+}
+
+ARG=$1
+echo "ARG: " $ARG
+
+
+if [ "$ARG" = "travis-CI" ]; then
+    build_frontend_fake
+else
+    build_frontend
+fi
+
+clean_env
+build_backend
+build_onnx_graph
+package

scripts/setup_dev_env.sh

+#!/bin/bash
+set -ex
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+echo "Setting up nodejs dependencies"
+cd $SCRIPT_DIR/../frontend
+npm install
+
+processors=1
+if [ "$(uname)" == "Darwin" ]; then
+    processors=`sysctl -n hw.ncpu`
+elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
+    processors=`nproc`
+fi
+
+echo "Building VisualDL SDK"
+cd $SCRIPT_DIR/..
+mkdir -p build
+cd build
+cmake ..
+make -j $processors
+
+export PYTHON_PATH=$PYTHON_PATH:"$SCRIPT_DIR/.."

scripts/start_dev_server.sh

+#!/bin/bash
+set -ex
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd $SCRIPT_DIR/../frontend
+./node_modules/.bin/webpack --watch --config tool/webpack.dev.config.js --output-path=../visualdl/server/dist &
+
+export PYTHON_PATH=$PYTHON_PATH:"$SCRIPT_DIR/.."
+
+cd $SCRIPT_DIR/../visualdl/server
+python visualDL "$@"

scripts/tests.sh

+#!/bin/bash
+set -ex
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $SCRIPT_DIR/..
+
+mode=$1
+readonly TOP_DIR=$(pwd)
+readonly core_path=$TOP_DIR/build/visualdl/logic
+readonly python_path=$TOP_DIR/visualdl/python
+readonly max_file_size=1000000 # 1MB
+# version number follow the rule of https://semver.org/
+readonly version_number=`cat VERSION_NUMBER | sed 's/\([0-9]*.[0-9]*.[0-9]*\).*/\1/g'`
+
+sudo="sudo"
+
+if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo=""; fi
+
+if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+		curl -O http://python-distribute.org/distribute_setup.py
+		python distribute_setup.py
+		curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
+		python get-pip.py
+fi
+
+$sudo pip install numpy
+$sudo pip install Flask
+$sudo pip install Pillow
+$sudo pip install protobuf
+
+export PYTHONPATH="${core_path}:${python_path}"
+
+# install the visualdl wheel first
+package() {
+    # some bug with frontend build
+    # a environment variable to skip frontend build
+    export VS_BUILD_MODE="travis-CI"
+
+    cd $TOP_DIR/visualdl/server
+    # manully install protobuf3
+    curl -OL https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip
+    unzip protoc-3.1.0-linux-x86_64.zip -d protoc3
+    export PATH="$PATH:$(pwd)/protoc3/bin"
+    chmod +x protoc3/bin/*
+
+
+    cd $TOP_DIR
+    python setup.py bdist_wheel
+    $sudo pip install dist/visualdl-${version_number}*.whl
+}
+
+backend_test() {
+    cd $TOP_DIR
+    mkdir -p build
+    cd build
+    cmake ..
+    make
+    make test
+}
+
+frontend_test() {
+    cd $TOP_DIR
+    cd frontend
+    npm install
+    npm run build
+}
+
+server_test() {
+    $sudo pip install google
+    $sudo pip install protobuf==3.1.0
+
+    cd $TOP_DIR/visualdl/server
+    bash graph_test.sh
+
+    cd $TOP_DIR/visualdl/server
+    python lib_test.py
+}
+
+# check the size of files in the repo.
+# reject PR that has some big data included.
+bigfile_reject() {
+    cd $TOP_DIR
+    # it failed to exclude .git, remove it first.
+    rm -rf .git
+    local largest_file=$(find . -path .git -prune -o -printf '%s %p\n' | sort -nr | grep -v "CycleGAN"| head -n1)
+    local size=$(echo "$largest_file" | awk '{print $1}')
+    if [ "$size" -ge "$max_file_size" ]; then
+        echo $largest_file
+        echo "file size exceed $max_file_size"
+        echo "Should not add large data or binary file."
+        exit -1
+    fi
+}
+
+echo "mode" $mode
+
+if [ $mode = "backend" ]; then
+    backend_test
+elif [ $mode = "all" ]; then
+    # bigfile_reject should be tested first, or some files downloaded may fail this test.
+    bigfile_reject
+    package
+    frontend_test
+    backend_test
+    server_test
+elif [ $mode = "local" ]; then
+    #frontend_test
+    backend_test
+    server_test
+else
+    frontend_test
+fi