diff --git a/frontend/packages/core/build.d.ts b/frontend/packages/core/build.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..b1354473d6cf012b9acca3c472cc65f9040c33ea --- /dev/null +++ b/frontend/packages/core/build.d.ts @@ -0,0 +1,2 @@ +export default function (action: string, ...args: string[]): Promise; +export const projectRoot: string; diff --git a/frontend/packages/core/build.ts b/frontend/packages/core/build.js similarity index 94% rename from frontend/packages/core/build.ts rename to frontend/packages/core/build.js index dce44b7edb6596bc6520851a049ee554fff2d26d..6885bb890c4a82a31889ae9c3563e9b218da6fbc 100644 --- a/frontend/packages/core/build.ts +++ b/frontend/packages/core/build.js @@ -8,7 +8,7 @@ import {writeFileSync} from 'fs'; const next = require.resolve('next/dist/bin/next'); export const projectRoot = path.dirname(require.resolve('@visualdl/core')); -export default function (action: string, ...args: string[]): Promise { +export default function (action, ...args) { return new Promise((resolve, reject) => { const capitalizedAction = action.replace(/^./, w => w.toUpperCase()); diff --git a/frontend/packages/core/pages/_document.tsx b/frontend/packages/core/pages/_document.tsx index 4d2c582fcfe3af2bb4c622d437b3c9bf6e9b59e1..8592a12b6b8ac4a56b84a4bf3e76cb616ac488b5 100644 --- a/frontend/packages/core/pages/_document.tsx +++ b/frontend/packages/core/pages/_document.tsx @@ -48,6 +48,9 @@ export default class VDLDocument extends Document { +
diff --git a/frontend/packages/core/utils/publicPath.ts b/frontend/packages/core/utils/publicPath.ts new file mode 100644 index 0000000000000000000000000000000000000000..bad3f43d8116723ad494206a1dadeb02545a4de9 --- /dev/null +++ b/frontend/packages/core/utils/publicPath.ts @@ -0,0 +1,7 @@ +// eslint-disable-next-line @typescript-eslint/camelcase +declare let __webpack_public_path__: string; + +// eslint-disable-next-line prefer-const, @typescript-eslint/camelcase +__webpack_public_path__ = '/test/'; + +export {}; diff --git a/frontend/packages/i18n/src/config/create-config.ts b/frontend/packages/i18n/src/config/create-config.ts index cfe3956177416aef730f3fc29f6fc567d18c0d7b..74d425d045236b5998d69130b10244b95456927b 100644 --- a/frontend/packages/i18n/src/config/create-config.ts +++ b/frontend/packages/i18n/src/config/create-config.ts @@ -96,9 +96,10 @@ export const createConfig = (userConfig: Config): Config => { /* Set client side backend */ + const publicPath = (window as any).__vdl_public_path__ || ''; combinedConfig.backend = { - loadPath: `${process.env.PUBLIC_PATH}/${clientLocalePath}/${localeStructure}.${localeExtension}`, - addPath: `${process.env.PUBLIC_PATH}/${clientLocalePath}/${localeStructure}.missing.${localeExtension}` + loadPath: `${publicPath}/${clientLocalePath}/${localeStructure}.${localeExtension}`, + addPath: `${publicPath}/${clientLocalePath}/${localeStructure}.missing.${localeExtension}` }; combinedConfig.ns = [combinedConfig.defaultNS]; diff --git a/frontend/packages/serverless/package.json b/frontend/packages/serverless/package.json index 06b39555f183bf70e7b7a0714327b7784c6d125b..0fa49114ca65e5acdca11a365b17f203ed1245d7 100644 --- a/frontend/packages/serverless/package.json +++ b/frontend/packages/serverless/package.json @@ -27,7 +27,7 @@ "dist" ], "scripts": { - "build": "cross-env PUBLIC_PATH=/app API_URL=/api ts-node build.ts", + "build": "cross-env ts-node build.ts", "test": "echo \"Error: no test specified\" && exit 0" }, "devDependencies": { diff --git a/frontend/scripts/build.sh b/frontend/scripts/build.sh index 8aa7c59b39b21fa4afa0456ff8576c04ff4a9953..802185ac1044a306b4f72c8402bc372ccd1d4312 100755 --- a/frontend/scripts/build.sh +++ b/frontend/scripts/build.sh @@ -2,7 +2,7 @@ set -e -WORKING_PATH=`pwd` +WORKING_PATH=$(pwd) SERVER_DIR="packages/server/dist" SERVER_DIR_PATH="$WORKING_PATH/$SERVER_DIR" SERVERLESS_DIR="packages/serverless/dist" @@ -10,15 +10,33 @@ SERVERLESS_DIR_PATH="$WORKING_PATH/$SERVERLESS_DIR" OUTPUT="output" OUTPUT_PATH="$WORKING_PATH/$OUTPUT" +# clean +rm -rf "$SERVER_DIR_PATH" +rm -rf "$SERVERLESS_DIR_PATH" + # build -npx lerna run build +if [ "$SCOPE" = "serverless" ]; then + npx lerna run --scope "@visualdl/serverless" --include-dependencies build +elif [ "$SCOPE" = "server" ]; then + npx lerna run --scope "@visualdl/server" --include-dependencies build +elif [ "$SCOPE" = "cli" ]; then + npx lerna run --scope "@visualdl/cli" --include-dependencies build +elif [ "$SCOPE" = "app" ]; then + npx lerna run --scope "@visualdl/app" --include-dependencies build +else + npx lerna run build +fi # generate output -rm -rf ${OUTPUT_PATH} -mkdir -p ${OUTPUT_PATH} +rm -rf "$OUTPUT_PATH" +mkdir -p "$OUTPUT_PATH" # package server files -(cd ${SERVER_DIR_PATH} && tar zcf ${OUTPUT_PATH}/server.tar.gz .) +if [ -d "$SERVER_DIR_PATH" ]; then + (cd "$SERVER_DIR_PATH" && tar zcf "${OUTPUT_PATH}/server.tar.gz" .) +fi # package serverless files -(cd ${SERVERLESS_DIR_PATH} && tar zcf ${OUTPUT_PATH}/serverless.tar.gz .) +if [ -d "$SERVERLESS_DIR_PATH" ]; then + (cd "$SERVERLESS_DIR_PATH" && tar zcf "${OUTPUT_PATH}/serverless.tar.gz" .) +fi diff --git a/frontend/scripts/install.sh b/frontend/scripts/install.sh index f88861ca4884ce9dfda9cf2a5a3f9df4b2d5df83..9277b832eb5337877e854e36fe484c72adff115e 100755 --- a/frontend/scripts/install.sh +++ b/frontend/scripts/install.sh @@ -6,7 +6,7 @@ set -e # https://rustup.rs/ if ! hash rustup 2>/dev/null; then curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path --default-toolchain nightly -y - source $HOME/.cargo/env + source "$HOME/.cargo/env" fi @@ -20,23 +20,28 @@ fi # wine if hash apt 2>/dev/null; then - sudo dpkg --add-architecture i386 - wget -nc https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_18.04/Release.key - sudo apt-key add Release.key - sudo apt-add-repository 'deb https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_18.04/ ./' - - wget -nc https://dl.winehq.org/wine-builds/winehq.key - sudo apt-key add winehq.key - sudo apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ bionic main' - - sudo apt update - sudo apt install --install-recommends winehq-stable + SYSTEM=$(uname -s); + if [ "$SYSTEM" = "Linux" ]; then + sudo dpkg --add-architecture i386 + wget -nc https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_18.04/Release.key + sudo apt-key add Release.key + sudo apt-add-repository 'deb https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_18.04/ ./' + + wget -nc https://dl.winehq.org/wine-builds/winehq.key + sudo apt-key add winehq.key + sudo apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ bionic main' + + sudo apt update + sudo apt install --install-recommends winehq-stable + fi fi # yarn -curl --compressed -o- -L https://yarnpkg.com/install.sh | bash -export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH" +if ! hash yarn 2>/dev/null; then + curl --compressed -o- -L https://yarnpkg.com/install.sh | bash + export PATH="$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin:$PATH" +fi # yarn install yarn install --frozen-lockfile diff --git a/requirements.txt b/requirements.txt index 6e2900a60fa75e3812efe0e3b26fd1b49004c5a0..abe5de831b8f2ef1bc30a081de57b41ae7cbbdb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,7 @@ -numpy == 1.16.6; python_version < "3.0" -numpy ; python_version >= "3.0" -recommonmark >= 0.6.0 +numpy +requests scipy == 1.2.3; python_version < "3.0" scipy >= 1.4.1; python_version >= "3.0" -Sphinx == 1.8.5; python_version < "3.0" -Sphinx >= 2.4.4; python_version >= "3.0" -sphinx-rtd-theme >= 0.4.3 flake8 >= 3.7.9 Pillow == 6.2.2; python_version < "3.0" Pillow >= 7.0.0; python_version >= "3.0" @@ -14,4 +10,4 @@ flask >= 1.1.1 Flask-Babel >= 1.0.0 six >= 1.14.0 protobuf >= 3.1.0 -opencv-python +opencv-python \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh index 7050a3bbd82fd4856db2bb915c13ddaf079643c0..85a82f584e1b47671c9b2b39384ed14c16bb1146 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -12,80 +12,24 @@ build_frontend_fake() { } build_frontend_from_source() { - build_frontend_fake + mkdir -p "$BUILD_DIR/package/dist" cd "$FRONTEND_DIR" ./scripts/install.sh - ./scripts/build.sh + SCOPE="serverless" PUBLIC_PATH="/{{PUBLIC_PATH}}" API_URL="/{{PUBLIC_PATH}}/api" ./scripts/build.sh # extract - tar zxf "$FRONTEND_DIR/output/serverless.tar.gz" -C "$BUILD_DIR/package/serverless" + tar zxf "$FRONTEND_DIR/output/serverless.tar.gz" -C "$BUILD_DIR/package/dist" } build_frontend() { - local PACKAGE="@visualdl/serverless" - local NAME=${PACKAGE#*@} - local NAME=${NAME////-} - echo ${NAME} - local TAG="latest" - local TARBALL="${PACKAGE}@${TAG}" - - # get version - local VERSION - VERSION=$(npm view ${TARBALL} dist-tags.${TAG}) - # shellcheck disable=SC2181 - if [[ "$?" -ne "0" ]]; then - echo "Cannot get version" - exit 1 - fi - local FILENAME="${NAME}-${VERSION}.tgz" - - # get sha1sum - local SHA1SUM; - SHA1SUM=$(npm view ${TARBALL} dist.shasum) - # shellcheck disable=SC2181 - if [[ "$?" -ne "0" ]]; then - echo "Cannot get sha1sum" - exit 1 - fi - rm -f "$BUILD_DIR/${NAME}-*.tgz.sha1" - echo "${SHA1SUM} ${FILENAME}" > "$BUILD_DIR/${FILENAME}.sha1" - - local DOWNLOAD="1" - # cached file exists - if [[ -f "$BUILD_DIR/$FILENAME" ]]; then - # check sha1sum - (cd "$BUILD_DIR" && sha1sum -c "${FILENAME}.sha1") - # check pass, use cached file - # shellcheck disable=SC2181 - if [[ "$?" -eq "0" ]]; then - echo "Using cached npm package file ${FILENAME}" - DOWNLOAD="0" - fi - fi - - if [[ "$DOWNLOAD" -eq "1" ]]; then - echo "Downloading npm package, please wait..." - - # remove cache - rm -f "$BUILD_DIR/${NAME}-*.tgz" - - # download file - FILENAME=$( (cd "$BUILD_DIR" && npm pack ${TARBALL}) ) + mkdir -p "$BUILD_DIR/package/dist" - # check sha1sum of downloaded file - (cd "BUILD_DIR" && sha1sum -c "${FILENAME}.sha1") - # shellcheck disable=SC2181 - if [[ "$?" -ne "0" ]]; then - echo "Check sum failed, download may not finish correctly." - exit 1 - else - echo "Check sum pass." - fi - fi + cd "$FRONTEND_DIR/packages/serverless" + npm install --no-package-lock + npm run build - # extract - tar zxf "$BUILD_DIR/$FILENAME" -C "$BUILD_DIR" + cp -a dist/. "$BUILD_DIR/package/dist/" } clean_env() { @@ -110,6 +54,9 @@ if [[ "$ARG" = "travis-CI" ]]; then build_frontend_fake elif [[ "$ARG" = "from-source" ]]; then build_frontend_from_source +elif [[ "$ARG" = "no-build" ]]; then + build_frontend_fake + echo "skipping build frontend" else build_frontend fi diff --git a/setup.py b/setup.py index 22f3e30ed5bd07a29d651395b868043da32eeeb5..33e5c0ed7d472af0da924780480c27bf5c8b49eb 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ TOP_DIR = os.path.realpath(os.path.dirname(__file__)) PYTHON_SDK_DIR = os.path.join(TOP_DIR, 'visualdl/python') BUILD_DIR = os.path.join(TOP_DIR, 'build') MODE = os.environ.get('VS_BUILD_MODE', 'RELEASE') +FRONTEND = os.environ.get('BUILD_FRONTEND') def read(name): @@ -83,6 +84,10 @@ class build_py(setuptools.command.build_py.build_py): env = dict(os.environ) if MODE == "travis-CI": cmd.append('travis-CI') + elif FRONTEND == "source": + cmd.append('from-source') + elif FRONTEND == "none": + cmd.append('no-build') if sys.version_info[0] >= 3: env["WITH_PYTHON3"] = "ON" subprocess.check_call(cmd, env=env) @@ -104,7 +109,8 @@ setup( long_description_content_type='text/markdown', install_requires=REQUIRED_PACKAGES, package_data={ - 'visualdl.server': [('dist' + ('/*' * n)) for n in range(1, 20)], + 'visualdl.server': [('dist' + ('/*' * n)) for n in range(1, 20)] + [('static' + ('/*' * n)) for n in + range(1, 20)], 'visualdl.python': ['dog.jpg', 'testing.wav'] }, packages=find_packages(), diff --git a/visualdl/server/app.py b/visualdl/server/app.py index 39a07d6f2d629e9b241f5de28f2818907571cc8e..1befc7d59e4c712b0ae6ab03afcbaa5710500e60 100644 --- a/visualdl/server/app.py +++ b/visualdl/server/app.py @@ -27,12 +27,11 @@ import requests from visualdl.reader.reader import LogReader from argparse import ArgumentParser -from flask import (Flask, Response, redirect, request, send_file, - send_from_directory) +from flask import (Flask, Response, redirect, request, send_file, send_from_directory) from flask_babel import Babel import visualdl.server -from visualdl.server import lib +from visualdl.server import (lib, template) from visualdl.server.log import logger from visualdl.python.cache import MemCache @@ -45,7 +44,8 @@ support_language = ["en", "zh"] default_language = support_language[0] server_path = os.path.abspath(os.path.dirname(sys.argv[0])) -static_file_path = os.path.join(SERVER_DIR, "./dist") +static_file_path = os.path.join(SERVER_DIR, "./static") +template_file_path = os.path.join(SERVER_DIR, "./dist") mock_data_path = os.path.join(SERVER_DIR, "./mock_data/") @@ -56,13 +56,15 @@ class ParseArgs(object): port=8040, model_pb="", cache_timeout=20, - language=None): + language=None, + public_path=None): self.logdir = logdir self.host = host self.port = port self.model_pb = model_pb self.cache_timeout = cache_timeout self.language = language + self.public_path = public_path def try_call(function, *args, **kwargs): @@ -120,6 +122,14 @@ def parse_args(): type=str, action="store", help="set the default language") + parser.add_argument( + "-P", + "--public-path", + type=str, + action="store", + default="/app", + help="set public path" + ) args = parser.parse_args() if not args.logdir: @@ -155,94 +165,93 @@ def create_app(args): CACHE = MemCache(timeout=args.cache_timeout) cache_get = lib.cache_get(CACHE) + public_path = args.public_path.rstrip('/') + api_path = public_path + '/api' + @babel.localeselector def get_locale(): - language = args.language - if not language or language not in support_language: - language = request.accept_languages.best_match(support_language) - return language + lang = args.language + if not lang or lang not in support_language: + lang = request.accept_languages.best_match(support_language) + return lang - @app.route("/") + @app.route(public_path + "/") def index(): - language = get_locale() - if language == default_language: - return redirect('/app/index', code=302) - return redirect('/app/' + language + '/index', code=302) + lang = get_locale() + if lang == default_language: + return redirect(public_path + '/index', code=302) + return redirect(public_path + '/' + lang + '/index', code=302) - @app.route('/app/') + @app.route(public_path + '/') def serve_static(filename): return send_from_directory( os.path.join(server_path, static_file_path), filename if re.search(r'\..+$', filename) else filename + '.html') - @app.route('/graphs/image') - def serve_graph(): - return send_file(os.path.join(os.getcwd(), graph_image_path)) - - @app.route('/api/logdir') + @app.route(api_path + '/logdir') def logdir(): result = gen_result(0, "", {"logdir": args.logdir}) return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/language') + @app.route(api_path + '/language') def language(): data = get_locale() result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route("/api/components") + @app.route(api_path + "/components") def components(): data = cache_get('/data/components', lib.get_components, log_reader) result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/runs') + @app.route(api_path + '/runs') def runs(): data = cache_get('/data/runs', lib.get_runs, log_reader) result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/tags') + @app.route(api_path + '/tags') def tags(): data = cache_get('/data/tags', lib.get_tags, log_reader) result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/logs') + @app.route(api_path + '/logs') def logs(): data = cache_get('/data/logs', lib.get_logs, log_reader) result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route("/api/scalars/tags") + @app.route(api_path + "/scalars/tags") def scalar_tags(): data = cache_get("/data/plugin/scalars/tags", try_call, lib.get_scalar_tags, log_reader) result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route("/api/images/tags") + @app.route(api_path + "/images/tags") def image_tags(): data = cache_get("/data/plugin/images/tags", try_call, lib.get_image_tags, log_reader) result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route("/api/audio/tags") + @app.route(api_path + "/audio/tags") def audio_tags(): data = cache_get("/data/plugin/audio/tags", try_call, lib.get_audio_tags, log_reader) result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route("/api/embeddings/tags") + @app.route(api_path + "/embeddings/tags") def embeddings_tags(): data = cache_get("/data/plugin/embeddings/tags", try_call, lib.get_embeddings_tags, log_reader) result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/scalars/list') + @app.route(api_path + '/scalars/list') def scalars(): run = request.args.get('run') tag = request.args.get('tag') @@ -251,7 +260,7 @@ def create_app(args): result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/images/list') + @app.route(api_path + '/images/list') def images(): mode = request.args.get('run') tag = request.args.get('tag') @@ -263,7 +272,7 @@ def create_app(args): return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/images/image') + @app.route(api_path + '/images/image') def individual_image(): mode = request.args.get('run') tag = request.args.get('tag') # include a index @@ -275,7 +284,7 @@ def create_app(args): mode, tag, step_index) return Response(data, mimetype="image/png") - @app.route('/api/embeddings/embedding') + @app.route(api_path + '/embeddings/embedding') def embeddings(): run = request.args.get('run') tag = request.args.get('tag', 'default') @@ -288,7 +297,7 @@ def create_app(args): result = gen_result(0, "", data) return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/audio/list') + @app.route(api_path + '/audio/list') def audio(): run = request.args.get('run') tag = request.args.get('tag') @@ -300,7 +309,7 @@ def create_app(args): return Response(json.dumps(result), mimetype='application/json') - @app.route('/api/audio/audio') + @app.route(api_path + '/audio/audio') def individual_audio(): run = request.args.get('run') tag = request.args.get('tag') # include a index @@ -319,6 +328,7 @@ def create_app(args): def _open_browser(app, index_url): while True: + # noinspection PyBroadException try: requests.get(index_url) break @@ -333,6 +343,7 @@ def _run(logdir, model_pb="", cache_timeout=20, language=None, + public_path="/app", open_browser=False): args = ParseArgs( logdir=logdir, @@ -340,10 +351,11 @@ def _run(logdir, port=port, model_pb=model_pb, cache_timeout=cache_timeout, - language=language) + language=language, + public_path=public_path) logger.info(" port=" + str(args.port)) app = create_app(args) - index_url = "http://" + host + ":" + str(port) + index_url = "http://" + host + ":" + str(port) + args.public_path if open_browser: threading.Thread( target=_open_browser, kwargs={"app": app, @@ -357,6 +369,7 @@ def run(logdir, model_pb="", cache_timeout=20, language=None, + public_path="/app", open_browser=False): kwarg = { "logdir": logdir, @@ -365,6 +378,7 @@ def run(logdir, "model_pb": model_pb, "cache_timeout": cache_timeout, "language": language, + "public_path": public_path, "open_browser": open_browser } @@ -375,13 +389,16 @@ def run(logdir, def main(): args = parse_args() + template.render( + template_file_path, + static_file_path, + PUBLIC_PATH=args.public_path.strip('/')) logger.info(" port=" + str(args.port)) app = create_app(args=args) app.run(debug=False, host=args.host, port=args.port, threaded=False) if __name__ == "__main__": - args = parse_args() logger.info(" port=" + str(args.port)) app = create_app(args=args) diff --git a/visualdl/server/template.py b/visualdl/server/template.py new file mode 100644 index 0000000000000000000000000000000000000000..ebc1d5aba7fa5e2840c6cbc124952d712896bf01 --- /dev/null +++ b/visualdl/server/template.py @@ -0,0 +1,34 @@ +# Copyright (c) 2017 VisualDL Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ======================================================================= + +import os +from shutil import (copytree, rmtree) + + +def render(path, dest, **context): + if os.path.exists(dest): + rmtree(dest) + copytree(path, dest) + for root, dirs, files in os.walk(dest): + for file in files: + if file.endswith(".html") or file.endswith(".js") or file.endswith(".css"): + file_path = os.path.join(root, file) + content = "" + with open(file_path, "r") as f: + content = f.read() + for key, value in context.items(): + content = content.replace("{{" + key + "}}", value) + with open(file_path, "w") as f: + f.write(content)