feat: support changing public path when start

5304af1f · Peter Pan · 95c02462 · 5304af1f · 5304af1f · 5304af1f
13 changed file
--- a/frontend/packages/core/build.d.ts
+++ b/frontend/packages/core/build.d.ts
+export default function (action: string, ...args: string[]): Promise<number>;
+export const projectRoot: string;
--- a/frontend/packages/core/build.ts
+++ b/frontend/packages/core/build.ts
@@ -8,7 +8,7 @@ import {writeFileSync} from 'fs';
 const next = require.resolve('next/dist/bin/next');
 export const projectRoot = path.dirname(require.resolve('@visualdl/core'));

-export default function (action: string, ...args: string[]): Promise<number> {
+export default function (action, ...args) {
    return new Promise((resolve, reject) => {
        const capitalizedAction = action.replace(/^./, w => w.toUpperCase());


--- a/frontend/packages/core/pages/_document.tsx
+++ b/frontend/packages/core/pages/_document.tsx
@@ -48,6 +48,9 @@ export default class VDLDocument extends Document<VDLDocumentProps> {
            <Html lang={language} dir={languageDir}>
                <Head />
                <body>
+                    <script
+                        dangerouslySetInnerHTML={{__html: `__vdl_public_path__='${process.env.PUBLIC_PATH}'`}}
+                    ></script>
                    <Main />
                    <NextScript />
                </body>

--- a/frontend/packages/core/utils/publicPath.ts
+++ b/frontend/packages/core/utils/publicPath.ts
+// eslint-disable-next-line @typescript-eslint/camelcase
+declare let __webpack_public_path__: string;
+
+// eslint-disable-next-line prefer-const, @typescript-eslint/camelcase
+__webpack_public_path__ = '/test/';
+
+export {};
--- a/frontend/packages/i18n/src/config/create-config.ts
+++ b/frontend/packages/i18n/src/config/create-config.ts
@@ -96,9 +96,10 @@ export const createConfig = (userConfig: Config): Config => {
        /*
            Set client side backend
        */
+        const publicPath = (window as any).__vdl_public_path__ || '';
        combinedConfig.backend = {
-            loadPath: `${process.env.PUBLIC_PATH}/${clientLocalePath}/${localeStructure}.${localeExtension}`,
-            addPath: `${process.env.PUBLIC_PATH}/${clientLocalePath}/${localeStructure}.missing.${localeExtension}`
+            loadPath: `${publicPath}/${clientLocalePath}/${localeStructure}.${localeExtension}`,
+            addPath: `${publicPath}/${clientLocalePath}/${localeStructure}.missing.${localeExtension}`
        };

        combinedConfig.ns = [combinedConfig.defaultNS];

--- a/frontend/packages/serverless/package.json
+++ b/frontend/packages/serverless/package.json
@@ -27,7 +27,7 @@
        "dist"
    ],
    "scripts": {
-        "build": "cross-env PUBLIC_PATH=/app API_URL=/api ts-node build.ts",
+        "build": "cross-env ts-node build.ts",
        "test": "echo \"Error: no test specified\" && exit 0"
    },
    "devDependencies": {

--- a/frontend/scripts/build.sh
+++ b/frontend/scripts/build.sh
@@ -2,7 +2,7 @@

 set -e

-WORKING_PATH=`pwd`
+WORKING_PATH=$(pwd)
 SERVER_DIR="packages/server/dist"
 SERVER_DIR_PATH="$WORKING_PATH/$SERVER_DIR"
 SERVERLESS_DIR="packages/serverless/dist"
@@ -10,15 +10,33 @@ SERVERLESS_DIR_PATH="$WORKING_PATH/$SERVERLESS_DIR"
 OUTPUT="output"
 OUTPUT_PATH="$WORKING_PATH/$OUTPUT"

+# clean
+rm -rf "$SERVER_DIR_PATH"
+rm -rf "$SERVERLESS_DIR_PATH"
+
 # build
-npx lerna run build
+if [ "$SCOPE" = "serverless" ]; then
+    npx lerna run --scope "@visualdl/serverless" --include-dependencies build
+elif [ "$SCOPE" = "server" ]; then
+    npx lerna run --scope "@visualdl/server" --include-dependencies build
+elif [ "$SCOPE" = "cli" ]; then
+    npx lerna run --scope "@visualdl/cli" --include-dependencies build
+elif [ "$SCOPE" = "app" ]; then
+    npx lerna run --scope "@visualdl/app" --include-dependencies build
+else
+    npx lerna run build
+fi

 # generate output
-rm -rf ${OUTPUT_PATH}
-mkdir -p ${OUTPUT_PATH}
+rm -rf "$OUTPUT_PATH"
+mkdir -p "$OUTPUT_PATH"

 # package server files
-(cd ${SERVER_DIR_PATH} && tar zcf ${OUTPUT_PATH}/server.tar.gz .)
+if [ -d "$SERVER_DIR_PATH" ]; then
+    (cd "$SERVER_DIR_PATH" && tar zcf "${OUTPUT_PATH}/server.tar.gz" .)
+fi

 # package serverless files
-(cd ${SERVERLESS_DIR_PATH} && tar zcf ${OUTPUT_PATH}/serverless.tar.gz .)
+if [ -d "$SERVERLESS_DIR_PATH" ]; then
+    (cd "$SERVERLESS_DIR_PATH" && tar zcf "${OUTPUT_PATH}/serverless.tar.gz" .)
+fi
--- a/frontend/scripts/install.sh
+++ b/frontend/scripts/install.sh
@@ -6,7 +6,7 @@ set -e
 # https://rustup.rs/
 if ! hash rustup 2>/dev/null; then
    curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --default-toolchain nightly -y
-    source $HOME/.cargo/env
+    source "$HOME/.cargo/env"
 fi


@@ -20,23 +20,28 @@ fi

 # wine
 if hash apt 2>/dev/null; then
-    sudo dpkg --add-architecture i386
-    wget -nc https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_18.04/Release.key
-    sudo apt-key add Release.key
-    sudo apt-add-repository 'deb https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_18.04/ ./'
-
-    wget -nc https://dl.winehq.org/wine-builds/winehq.key
-    sudo apt-key add winehq.key
-    sudo apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ bionic main'
-
-    sudo apt update
-    sudo apt install --install-recommends winehq-stable
+    SYSTEM=$(uname -s);
+    if [ "$SYSTEM" = "Linux" ]; then
+        sudo dpkg --add-architecture i386
+        wget -nc https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_18.04/Release.key
+        sudo apt-key add Release.key
+        sudo apt-add-repository 'deb https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_18.04/ ./'
+
+        wget -nc https://dl.winehq.org/wine-builds/winehq.key
+        sudo apt-key add winehq.key
+        sudo apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ bionic main'
+
+        sudo apt update
+        sudo apt install --install-recommends winehq-stable
+    fi
 fi


 # yarn
-curl --compressed -o- -L https://yarnpkg.com/install.sh | bash
-export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH"
+if ! hash yarn 2>/dev/null; then
+    curl --compressed -o- -L https://yarnpkg.com/install.sh | bash
+    export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH"
+fi

 # yarn install
 yarn install --frozen-lockfile
--- a/requirements.txt
+++ b/requirements.txt
-numpy == 1.16.6; python_version < "3.0"
-numpy ; python_version >= "3.0"
-recommonmark >= 0.6.0
+numpy
+requests
 scipy == 1.2.3; python_version < "3.0"
 scipy >= 1.4.1; python_version >= "3.0"
-Sphinx == 1.8.5; python_version < "3.0"
-Sphinx >= 2.4.4; python_version >= "3.0"
-sphinx-rtd-theme >= 0.4.3
 flake8 >= 3.7.9
 Pillow == 6.2.2; python_version < "3.0"
 Pillow >= 7.0.0; python_version >= "3.0"
@@ -14,4 +10,4 @@ flask >= 1.1.1
 Flask-Babel >= 1.0.0
 six >= 1.14.0
 protobuf >= 3.1.0
-opencv-python
+opencv-python
\ No newline at end of file
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -12,80 +12,24 @@ build_frontend_fake() {
 }

 build_frontend_from_source() {
-    build_frontend_fake
+    mkdir -p "$BUILD_DIR/package/dist"

    cd "$FRONTEND_DIR"
    ./scripts/install.sh
-    ./scripts/build.sh
+    SCOPE="serverless" PUBLIC_PATH="/{{PUBLIC_PATH}}" API_URL="/{{PUBLIC_PATH}}/api" ./scripts/build.sh

    # extract
-    tar zxf "$FRONTEND_DIR/output/serverless.tar.gz" -C "$BUILD_DIR/package/serverless"
+    tar zxf "$FRONTEND_DIR/output/serverless.tar.gz" -C "$BUILD_DIR/package/dist"
 }

 build_frontend() {
-    local PACKAGE="@visualdl/serverless"
-    local NAME=${PACKAGE#*@}
-    local NAME=${NAME////-}
-    echo ${NAME}
-    local TAG="latest"
-    local TARBALL="${PACKAGE}@${TAG}"
-
-    # get version
-    local VERSION
-    VERSION=$(npm view ${TARBALL} dist-tags.${TAG})
-    # shellcheck disable=SC2181
-    if [[ "$?" -ne "0" ]]; then
-        echo "Cannot get version"
-        exit 1
-    fi
-    local FILENAME="${NAME}-${VERSION}.tgz"
-
-    # get sha1sum
-    local SHA1SUM;
-    SHA1SUM=$(npm view ${TARBALL} dist.shasum)
-    # shellcheck disable=SC2181
-    if [[ "$?" -ne "0" ]]; then
-        echo "Cannot get sha1sum"
-        exit 1
-    fi
-    rm -f "$BUILD_DIR/${NAME}-*.tgz.sha1"
-    echo "${SHA1SUM} ${FILENAME}" > "$BUILD_DIR/${FILENAME}.sha1"
-
-    local DOWNLOAD="1"
-    # cached file exists
-    if [[ -f "$BUILD_DIR/$FILENAME" ]]; then
-        # check sha1sum
-        (cd "$BUILD_DIR" && sha1sum -c "${FILENAME}.sha1")
-        # check pass, use cached file
-        # shellcheck disable=SC2181
-        if [[ "$?" -eq "0" ]]; then
-            echo "Using cached npm package file ${FILENAME}"
-            DOWNLOAD="0"
-        fi
-    fi
-
-    if [[ "$DOWNLOAD" -eq "1" ]]; then
-        echo "Downloading npm package, please wait..."
-
-        # remove cache
-        rm -f "$BUILD_DIR/${NAME}-*.tgz"
-
-        # download file
-        FILENAME=$( (cd "$BUILD_DIR" && npm pack ${TARBALL}) )
+    mkdir -p "$BUILD_DIR/package/dist"

-        # check sha1sum of downloaded file
-        (cd "BUILD_DIR" && sha1sum -c "${FILENAME}.sha1")
-        # shellcheck disable=SC2181
-        if [[ "$?" -ne "0" ]]; then
-            echo "Check sum failed, download may not finish correctly."
-            exit 1
-        else
-            echo "Check sum pass."
-        fi
-    fi
+    cd "$FRONTEND_DIR/packages/serverless"
+    npm install --no-package-lock
+    npm run build

-    # extract
-    tar zxf "$BUILD_DIR/$FILENAME" -C "$BUILD_DIR"
+    cp -a dist/. "$BUILD_DIR/package/dist/"
 }

 clean_env() {
@@ -110,6 +54,9 @@ if [[ "$ARG" = "travis-CI" ]]; then
    build_frontend_fake
 elif [[ "$ARG" = "from-source" ]]; then
    build_frontend_from_source
+elif [[ "$ARG" = "no-build" ]]; then
+    build_frontend_fake
+    echo "skipping build frontend"
 else
    build_frontend
 fi

--- a/setup.py
+++ b/setup.py
@@ -31,6 +31,7 @@ TOP_DIR = os.path.realpath(os.path.dirname(__file__))
 PYTHON_SDK_DIR = os.path.join(TOP_DIR, 'visualdl/python')
 BUILD_DIR = os.path.join(TOP_DIR, 'build')
 MODE = os.environ.get('VS_BUILD_MODE', 'RELEASE')
+FRONTEND = os.environ.get('BUILD_FRONTEND')


 def read(name):
@@ -83,6 +84,10 @@ class build_py(setuptools.command.build_py.build_py):
        env = dict(os.environ)
        if MODE == "travis-CI":
            cmd.append('travis-CI')
+        elif FRONTEND == "source":
+            cmd.append('from-source')
+        elif FRONTEND == "none":
+            cmd.append('no-build')
        if sys.version_info[0] >= 3:
            env["WITH_PYTHON3"] = "ON"
        subprocess.check_call(cmd, env=env)
@@ -104,7 +109,8 @@ setup(
    long_description_content_type='text/markdown',
    install_requires=REQUIRED_PACKAGES,
    package_data={
-        'visualdl.server': [('dist' + ('/*' * n)) for n in range(1, 20)],
+        'visualdl.server': [('dist' + ('/*' * n)) for n in range(1, 20)] + [('static' + ('/*' * n)) for n in
+                                                                            range(1, 20)],
        'visualdl.python': ['dog.jpg', 'testing.wav']
    },
    packages=find_packages(),

--- a/visualdl/server/app.py
+++ b/visualdl/server/app.py
@@ -27,12 +27,11 @@ import requests
 from visualdl.reader.reader import LogReader
 from argparse import ArgumentParser

-from flask import (Flask, Response, redirect, request, send_file,
-                   send_from_directory)
+from flask import (Flask, Response, redirect, request, send_file, send_from_directory)
 from flask_babel import Babel

 import visualdl.server
-from visualdl.server import lib
+from visualdl.server import (lib, template)
 from visualdl.server.log import logger
 from visualdl.python.cache import MemCache

@@ -45,7 +44,8 @@ support_language = ["en", "zh"]
 default_language = support_language[0]

 server_path = os.path.abspath(os.path.dirname(sys.argv[0]))
-static_file_path = os.path.join(SERVER_DIR, "./dist")
+static_file_path = os.path.join(SERVER_DIR, "./static")
+template_file_path = os.path.join(SERVER_DIR, "./dist")
 mock_data_path = os.path.join(SERVER_DIR, "./mock_data/")


@@ -56,13 +56,15 @@ class ParseArgs(object):
                 port=8040,
                 model_pb="",
                 cache_timeout=20,
-                 language=None):
+                 language=None,
+                 public_path=None):
        self.logdir = logdir
        self.host = host
        self.port = port
        self.model_pb = model_pb
        self.cache_timeout = cache_timeout
        self.language = language
+        self.public_path = public_path


 def try_call(function, *args, **kwargs):
@@ -120,6 +122,14 @@ def parse_args():
        type=str,
        action="store",
        help="set the default language")
+    parser.add_argument(
+        "-P",
+        "--public-path",
+        type=str,
+        action="store",
+        default="/app",
+        help="set public path"
+    )

    args = parser.parse_args()
    if not args.logdir:
@@ -155,94 +165,93 @@ def create_app(args):
    CACHE = MemCache(timeout=args.cache_timeout)
    cache_get = lib.cache_get(CACHE)

+    public_path = args.public_path.rstrip('/')
+    api_path = public_path + '/api'
+
    @babel.localeselector
    def get_locale():
-        language = args.language
-        if not language or language not in support_language:
-            language = request.accept_languages.best_match(support_language)
-        return language
+        lang = args.language
+        if not lang or lang not in support_language:
+            lang = request.accept_languages.best_match(support_language)
+        return lang

-    @app.route("/")
+    @app.route(public_path + "/")
    def index():
-        language = get_locale()
-        if language == default_language:
-            return redirect('/app/index', code=302)
-        return redirect('/app/' + language + '/index', code=302)
+        lang = get_locale()
+        if lang == default_language:
+            return redirect(public_path + '/index', code=302)
+        return redirect(public_path + '/' + lang + '/index', code=302)

-    @app.route('/app/<path:filename>')
+    @app.route(public_path + '/<path:filename>')
    def serve_static(filename):
        return send_from_directory(
            os.path.join(server_path, static_file_path), filename
            if re.search(r'\..+$', filename) else filename + '.html')

-    @app.route('/graphs/image')
-    def serve_graph():
-        return send_file(os.path.join(os.getcwd(), graph_image_path))
-
-    @app.route('/api/logdir')
+    @app.route(api_path + '/logdir')
    def logdir():
        result = gen_result(0, "", {"logdir": args.logdir})
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/language')
+    @app.route(api_path + '/language')
    def language():
        data = get_locale()
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route("/api/components")
+    @app.route(api_path + "/components")
    def components():
        data = cache_get('/data/components', lib.get_components, log_reader)
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/runs')
+    @app.route(api_path + '/runs')
    def runs():
        data = cache_get('/data/runs', lib.get_runs, log_reader)
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/tags')
+    @app.route(api_path + '/tags')
    def tags():
        data = cache_get('/data/tags', lib.get_tags, log_reader)
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/logs')
+    @app.route(api_path + '/logs')
    def logs():
        data = cache_get('/data/logs', lib.get_logs, log_reader)
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route("/api/scalars/tags")
+    @app.route(api_path + "/scalars/tags")
    def scalar_tags():
        data = cache_get("/data/plugin/scalars/tags", try_call,
                         lib.get_scalar_tags, log_reader)
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route("/api/images/tags")
+    @app.route(api_path + "/images/tags")
    def image_tags():
        data = cache_get("/data/plugin/images/tags", try_call,
                         lib.get_image_tags, log_reader)
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route("/api/audio/tags")
+    @app.route(api_path + "/audio/tags")
    def audio_tags():
        data = cache_get("/data/plugin/audio/tags", try_call,
                         lib.get_audio_tags, log_reader)
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route("/api/embeddings/tags")
+    @app.route(api_path + "/embeddings/tags")
    def embeddings_tags():
        data = cache_get("/data/plugin/embeddings/tags", try_call,
                         lib.get_embeddings_tags, log_reader)
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/scalars/list')
+    @app.route(api_path + '/scalars/list')
    def scalars():
        run = request.args.get('run')
        tag = request.args.get('tag')
@@ -251,7 +260,7 @@ def create_app(args):
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/images/list')
+    @app.route(api_path + '/images/list')
    def images():
        mode = request.args.get('run')
        tag = request.args.get('tag')
@@ -263,7 +272,7 @@ def create_app(args):

        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/images/image')
+    @app.route(api_path + '/images/image')
    def individual_image():
        mode = request.args.get('run')
        tag = request.args.get('tag')  # include a index
@@ -275,7 +284,7 @@ def create_app(args):
                         mode, tag, step_index)
        return Response(data, mimetype="image/png")

-    @app.route('/api/embeddings/embedding')
+    @app.route(api_path + '/embeddings/embedding')
    def embeddings():
        run = request.args.get('run')
        tag = request.args.get('tag', 'default')
@@ -288,7 +297,7 @@ def create_app(args):
        result = gen_result(0, "", data)
        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/audio/list')
+    @app.route(api_path + '/audio/list')
    def audio():
        run = request.args.get('run')
        tag = request.args.get('tag')
@@ -300,7 +309,7 @@ def create_app(args):

        return Response(json.dumps(result), mimetype='application/json')

-    @app.route('/api/audio/audio')
+    @app.route(api_path + '/audio/audio')
    def individual_audio():
        run = request.args.get('run')
        tag = request.args.get('tag')  # include a index
@@ -319,6 +328,7 @@ def create_app(args):

 def _open_browser(app, index_url):
    while True:
+        # noinspection PyBroadException
        try:
            requests.get(index_url)
            break
@@ -333,6 +343,7 @@ def _run(logdir,
         model_pb="",
         cache_timeout=20,
         language=None,
+         public_path="/app",
         open_browser=False):
    args = ParseArgs(
        logdir=logdir,
@@ -340,10 +351,11 @@ def _run(logdir,
        port=port,
        model_pb=model_pb,
        cache_timeout=cache_timeout,
-        language=language)
+        language=language,
+        public_path=public_path)
    logger.info(" port=" + str(args.port))
    app = create_app(args)
-    index_url = "http://" + host + ":" + str(port)
+    index_url = "http://" + host + ":" + str(port) + args.public_path
    if open_browser:
        threading.Thread(
            target=_open_browser, kwargs={"app": app,
@@ -357,6 +369,7 @@ def run(logdir,
        model_pb="",
        cache_timeout=20,
        language=None,
+        public_path="/app",
        open_browser=False):
    kwarg = {
        "logdir": logdir,
@@ -365,6 +378,7 @@ def run(logdir,
        "model_pb": model_pb,
        "cache_timeout": cache_timeout,
        "language": language,
+        "public_path": public_path,
        "open_browser": open_browser
    }

@@ -375,13 +389,16 @@ def run(logdir,

 def main():
    args = parse_args()
+    template.render(
+        template_file_path,
+        static_file_path,
+        PUBLIC_PATH=args.public_path.strip('/'))
    logger.info(" port=" + str(args.port))
    app = create_app(args=args)
    app.run(debug=False, host=args.host, port=args.port, threaded=False)


 if __name__ == "__main__":
-
    args = parse_args()
    logger.info(" port=" + str(args.port))
    app = create_app(args=args)

--- a/visualdl/server/template.py
+++ b/visualdl/server/template.py
+# Copyright (c) 2017 VisualDL Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =======================================================================
+
+import os
+from shutil import (copytree, rmtree)
+
+
+def render(path, dest, **context):
+    if os.path.exists(dest):
+        rmtree(dest)
+    copytree(path, dest)
+    for root, dirs, files in os.walk(dest):
+        for file in files:
+            if file.endswith(".html") or file.endswith(".js") or file.endswith(".css"):
+                file_path = os.path.join(root, file)
+                content = ""
+                with open(file_path, "r") as f:
+                    content = f.read()
+                for key, value in context.items():
+                    content = content.replace("{{" + key + "}}", value)
+                with open(file_path, "w") as f:
+                    f.write(content)