build: build frontend from npm tarball (#582)

* v2.0.0-beta.12

* v2.0.0-beta.13

* fix: use new development server

* v2.0.0-beta.14

* build: build frontend from npm tarball

frontend/.gitignore

@@ -10,7 +10,7 @@
 
 # next.js
 /.next/
-/out/
+/serverless/
 
 # production
 /build

frontend/.prettierignore

 dist
-out
+serverless
 wasm
 output

frontend/Dockerfile

@@ -4,16 +4,17 @@ from node:12
 WORKDIR /usr/src/app
 
 ENV NODE_ENV production
+ENV PORT 8999
+ENV HOST 0.0.0.0
 
 COPY package.json .
 COPY yarn.lock .
 
-RUN npm i -g pm2
-
-RUN yarn
+RUN yarn install --frozen-lockfile
 
 COPY dist dist
 COPY public public
+COPY ecosystem.config.js ecosystem.config.js
 
 EXPOSE 8999
-CMD ["pm2-runtime", "dist/server/index.js"]
+CMD ["yarn", "start"]

frontend/cli.js

@@ -65,8 +65,7 @@ pm2.connect(err => {
     pm2.list((err, list) => {
         exitIfError(err, 2);
 
-        const appRoot = require('app-root-path');
-        const ecosystem = require(appRoot + '/ecosystem.config');
+        const ecosystem = require('./ecosystem.config');
 
         const app = ecosystem.apps[0];
         const host = argv.host || 'localhost';

frontend/ecosystem.config.js

 /* eslint-disable @typescript-eslint/no-var-requires */
 
-const {resolve} = require('app-root-path');
-
 module.exports = {
     apps: [
         {
             name: 'visualdl',
             script: 'dist/server/index.js',
-            cwd: resolve('/'),
+            cwd: __dirname,
             args: '',
             instances: 'max',
             autorestart: true,

frontend/index.js

-module.exports = require(__dirname + '/dist/server/index.js');

frontend/package.json

 {
     "name": "visualdl",
-    "version": "2.0.0-beta.11",
+    "version": "2.0.0-beta.14",
     "title": "VisualDL",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
@@ -30,20 +30,19 @@
         "build:server": "tsc --project server/tsconfig.json",
         "build:wasm": "wasm-pack build --release --out-dir dist --out-name index wasm",
         "build": "./scripts/build.sh",
-        "export": "next export",
-        "start": "NODE_ENV=production node dist/server/index.js",
+        "export": "next export -o serverless",
+        "start": "pm2-runtime ecosystem.config.js",
         "lint": "tsc -p tsconfig.json --noEmit && tsc -p server/tsconfig.json --noEmit && eslint --ext .tsx,.jsx.ts,.js --ignore-path .gitignore .",
         "format": "prettier --write \"**/*.ts\" \"**/*.tsx\" \"**/*.js\"  \"**/*.jsx\"",
         "test": "echo \"Error: no test specified\" && exit 0",
-        "prepublishOnly": "yarn lint && yarn test && ./scripts/prepare.sh",
+        "prepublishOnly": "yarn lint && yarn test && ./scripts/build.sh",
         "preversion": "yarn lint",
-        "version": "yarn format"
+        "version": "yarn format && git add -A"
     },
-    "main": "index.js",
     "bin": "cli.js",
     "files": [
         "dist",
-        "out",
+        "serverless",
         "public",
         "wasm/dist",
         "types",
@@ -52,7 +51,6 @@
         "ecosystem.config.js"
     ],
     "dependencies": {
-        "app-root-path": "3.0.0",
         "dagre-d3": "0.6.4",
         "detect-node": "2.0.4",
         "echarts": "4.6.0",

frontend/scripts/build.sh

@@ -9,41 +9,47 @@ fi
 WORKING_PATH=`pwd`
 SERVER_DIR="dist"
 SERVER_DIR_PATH="$WORKING_PATH/$SERVER_DIR"
-CLIENT_DIR="out"
-CLIENT_DIR_PATH="$WORKING_PATH/$CLIENT_DIR"
+SERVERLESS_DIR="serverless"
+SERVERLESS_DIR_PATH="$WORKING_PATH/$SERVERLESS_DIR"
 OUTPUT="output"
 OUTPUT_PATH="$WORKING_PATH/$OUTPUT"
 
-# generate dist
-rm -rf $SERVER_DIR_PATH
-mkdir -p $SERVER_DIR_PATH
-rm -rf $CLIENT_DIR_PATH
-mkdir -p $CLIENT_DIR_PATH
+build_server() {
+    # generate dist
+    rm -rf $SERVER_DIR_PATH
+    mkdir -p $SERVER_DIR_PATH
 
-# generate output
-rm -rf $OUTPUT_PATH
-mkdir -p $OUTPUT_PATH
+    # next build
+    yarn build:next
 
-# next build
-yarn build:next
+    # server build
+    yarn build:server
 
-# server build
-yarn build:server
+    # move static files
+    cp next.config.js $SERVER_DIR_PATH
+    cp package.json $SERVER_DIR_PATH
+    cp -r public $SERVER_DIR_PATH
+}
 
-# move static files
-cp next.config.js $SERVER_DIR_PATH
-cp package.json $SERVER_DIR_PATH
+build_serverless() {
+    build_server
 
-# package server files
-(cd $SERVER_DIR_PATH && tar zcf $OUTPUT_PATH/server.tar.gz .)
+    # generate dist
+    rm -rf $SERVERLESS_DIR_PATH
+    mkdir -p $SERVERLESS_DIR_PATH
 
-# export
-# WARNING: export LAST!!! dist files will be deleted by next after export
-yarn export
+    # next export
+    yarn export
+}
 
-# package client files
-(cd $CLIENT_DIR_PATH && tar zcf $OUTPUT_PATH/client.tar.gz .)
+# generate output
+rm -rf $OUTPUT_PATH
+mkdir -p $OUTPUT_PATH
+
+# package serverless files
+PUBLIC_PATH="/app" API_URL="/api" build_serverless
+(cd $SERVERLESS_DIR_PATH && tar zcf $OUTPUT_PATH/serverless.tar.gz .)
 
-# clean
-rm -rf $SERVER_DIR_PATH
-rm -rf $CLIENT_DIR_PATH
+# package server files
+PUBLIC_PATH="" API_URL="/api" build_server
+(cd $SERVER_DIR_PATH && tar zcf $OUTPUT_PATH/server.tar.gz .)

frontend/scripts/docker.sh

@@ -2,9 +2,6 @@
 
 set -e
 
-if [ ! -d dist ]; then
-    echo "Please build first!"
-    exit 1
-fi
+./scripts/build.sh
 
 docker build -t paddlepaddle/visualdl .

frontend/scripts/prepare.sh

-#!/bin/bash
-
-set -e
-
-if [ -f "$HOME/.cargo/env" ]; then
-    source $HOME/.cargo/env
-fi
-
-WORKING_PATH=`pwd`
-SERVER_DIR="dist"
-SERVER_DIR_PATH="$WORKING_PATH/$SERVER_DIR"
-
-# generate dist
-rm -rf $SERVER_DIR_PATH
-mkdir -p $SERVER_DIR_PATH
-
-# next build
-yarn build:next
-
-# server build
-yarn build:server
-
-# move static files
-cp next.config.js $SERVER_DIR_PATH
-cp package.json $SERVER_DIR_PATH

frontend/server/index.ts

@@ -15,7 +15,7 @@ const isDev = process.env.NODE_ENV !== 'production';
 setConfig(config);
 
 const host = process.env.HOST || 'localhost';
-const port: string | number = Number.parseInt(process.env.PORT, 10) || 8999;
+const port = Number.parseInt(process.env.PORT, 10) || 8999;
 const proxy = process.env.PROXY;
 const delay = Number.parseInt(process.env.DELAY, 10);
 
@@ -23,7 +23,7 @@ const server = express();
 const app = next({dev: isDev, conf: config});
 const handle = app.getRequestHandler();
 
-(async () => {
+async function start() {
     await app.prepare();
 
     if (proxy) {
@@ -59,6 +59,12 @@ const handle = app.getRequestHandler();
             });
         });
     });
-})();
+}
+
+if (require.main === module) {
+    start();
+}
+
+export default start;
 
 export {server, app};

frontend/utils/i18next/config/create-config.ts

@@ -32,6 +32,7 @@ export const createConfig = (userConfig: Config): Config => {
     if (isServer()) {
         const fs = eval("require('fs')");
         const path = require('path'); // eslint-disable-line @typescript-eslint/no-var-requires
+        const projectRoot = process.cwd();
         let serverLocalePath = localePath;
 
         /*
@@ -40,14 +41,14 @@ export const createConfig = (userConfig: Config): Config => {
         */
         if (typeof combinedConfig.defaultNS === 'string') {
             const defaultFile = `/${defaultLanguage}/${combinedConfig.defaultNS}.${localeExtension}`;
-            const defaultNSPath = path.join(process.cwd(), localePath, defaultFile);
+            const defaultNSPath = path.join(projectRoot, localePath, defaultFile);
             const defaultNSExists = fs.existsSync(defaultNSPath);
             if (!defaultNSExists) {
                 /*
                     If defaultNS doesn't exist, try to fall back to the deprecated static folder
                     https://github.com/isaachinman/next-i18next/issues/523
                 */
-                const staticDirPath = path.join(process.cwd(), STATIC_LOCALE_PATH, defaultFile);
+                const staticDirPath = path.join(projectRoot, STATIC_LOCALE_PATH, defaultFile);
                 const staticDirExists = fs.existsSync(staticDirPath);
 
                 if (staticDirExists) {
@@ -67,8 +68,8 @@ export const createConfig = (userConfig: Config): Config => {
             Set server side backend
         */
         combinedConfig.backend = {
-            loadPath: path.join(process.cwd(), `${serverLocalePath}/${localeStructure}.${localeExtension}`),
-            addPath: path.join(process.cwd(), `${serverLocalePath}/${localeStructure}.missing.${localeExtension}`)
+            loadPath: path.join(projectRoot, `${serverLocalePath}/${localeStructure}.${localeExtension}`),
+            addPath: path.join(projectRoot, `${serverLocalePath}/${localeStructure}.missing.${localeExtension}`)
         };
 
         /*
@@ -78,7 +79,7 @@ export const createConfig = (userConfig: Config): Config => {
         if (!combinedConfig.ns) {
             const getAllNamespaces = (p: string) =>
                 fs.readdirSync(p).map((file: string) => file.replace(`.${localeExtension}`, ''));
-            combinedConfig.ns = getAllNamespaces(path.join(process.cwd(), `${serverLocalePath}/${defaultLanguage}`));
+            combinedConfig.ns = getAllNamespaces(path.join(projectRoot, `${serverLocalePath}/${defaultLanguage}`));
         }
     } else {
         let clientLocalePath = localePath;

frontend/yarn.lock

@@ -1956,11 +1956,6 @@ anymatch@~3.1.1:
     normalize-path "^3.0.0"
     picomatch "^2.0.4"
 
-app-root-path@3.0.0:
-  version "3.0.0"
-  resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.0.0.tgz#210b6f43873227e18a4b810a032283311555d5ad"
-  integrity sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw==
-
 aproba@^1.1.1:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"

scripts/build.sh

@@ -8,14 +8,20 @@ BUILD_DIR=$TOP_DIR/build
 
 mkdir -p $BUILD_DIR
 
-build_frontend() {
+build_frontend_from_source() {
     cd $FRONTEND_DIR
     PUBLIC_PATH="/app" API_URL="/api" ./scripts/build.sh
 }
 
+build_frontend() {
+    local PACKAGE_NAME="visualdl"
+    local SRC=`npm view ${PACKAGE_NAME} dist.tarball`
+    wget $SRC -O "$BUILD_DIR/$PACKAGE_NAME.tar.gz"
+    tar zxf "$BUILD_DIR/$PACKAGE_NAME.tar.gz" -C "$BUILD_DIR"
+}
+
 build_frontend_fake() {
-    cd $FRONTEND_DIR
-    mkdir -p out
+    mkdir -p "$BUILD_DIR/package/serverless"
 }
 
 build_backend() {
@@ -46,7 +52,7 @@ clean_env() {
 
 package() {
     mkdir -p $TOP_DIR/visualdl/server/dist
-    cp -rf $FRONTEND_DIR/out/* $TOP_DIR/visualdl/server/dist
+    cp -rf $BUILD_DIR/package/serverless/* $TOP_DIR/visualdl/server/dist
     cp $BUILD_DIR/visualdl/logic/core.so $TOP_DIR/visualdl
     cp $BUILD_DIR/visualdl/logic/core.so $TOP_DIR/visualdl/python/
 }

scripts/setup_dev_env.sh

@@ -3,7 +3,9 @@ set -e
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-cd $SCRIPT_DIR/../frontend
+cd $SCRIPT_DIR/..
+mkdir -p build
+cd build
 
 which node >/dev/null 2>&1
 if [ $? -ne 0 ]; then
@@ -12,7 +14,7 @@ if [ $? -ne 0 ]; then
 fi
 
 echo "Setting up nodejs dependencies"
-npm install --no-package-lock
+npm install visualdl@latest --no-package-lock
 
 processors=1
 if [ "$(uname)" == "Darwin" ]; then
@@ -22,9 +24,6 @@ elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
 fi
 
 echo "Building VisualDL SDK"
-cd $SCRIPT_DIR/..
-mkdir -p build
-cd build
 cmake ..
 
 make -j $processors

scripts/start_dev_server.sh

@@ -41,16 +41,14 @@ CURRENT_DIR=`pwd`
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-cd $SCRIPT_DIR/../frontend
 export PYTHONPATH=$PYTHONPATH:"$SCRIPT_DIR/.."
 
 FRONTEND_PORT=8999
-PROXY="http://$HOST:$PORT" PUBLIC_PATH="/app" API_URL="/api" PORT=$FRONTEND_PORT yarn dev &
-# Track pid
-FRONTEND_PID=$!
+VDL_BIN="./build/node_modules/.bin/visualdl"
+$VDL_BIN start --port=$FRONTEND_PORT --host=$HOST --proxy="http://$HOST:$PORT"
 
 function finish {
-    kill -9 $FRONTEND_PID
+    $VDL_BIN stop
 }
 
 trap finish EXIT HUP INT QUIT PIPE TERM

setup.py

@@ -75,9 +75,9 @@ class BaseCommand(setuptools.Command):
 
 class build_py(setuptools.command.build_py.build_py):
     def run(self):
-        cmd = ['bash', 'build.sh']
+        cmd = ['bash', 'scripts/build.sh']
         if platform == "win32":
-            cmd = ['powershell', '-NoProfile', './build.ps1']
+            cmd = ['powershell', '-NoProfile', './scripts/build.ps1']
         env = dict(os.environ)
         if MODE == "travis-CI":
             cmd.append('travis-CI')