update demo (#899)

* docs: update demo

* publish fe@2.1.2

* chore: remove deprecated constants

* publish fe@2.1.3

* fix: fix web worker cross origin

* publish fe@2.1.4

* style: lint shell scripts

.pre-commit-config.yaml

@@ -17,6 +17,10 @@ repos:
       - id: flake8
         alias: be
         exclude: ^frontend/
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.7.1.1
+    hooks:
+      - id: shellcheck
   - repo: local
     hooks:
       - id: fe

frontend/lerna.json

@@ -2,7 +2,7 @@
     "packages": [
         "packages/*"
     ],
-    "version": "2.1.1",
+    "version": "2.1.4",
     "npmClient": "yarn",
     "useWorkspaces": true,
     "command": {

frontend/packages/cli/package.json

 {
     "name": "@visualdl/cli",
-    "version": "2.1.1",
+    "version": "2.1.4",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
         "visualdl",
@@ -34,7 +34,7 @@
         "dist"
     ],
     "dependencies": {
-        "@visualdl/server": "2.1.1",
+        "@visualdl/server": "2.1.4",
         "open": "7.3.0",
         "ora": "5.1.0",
         "pm2": "4.5.1",

frontend/packages/core/builder/environment.js

@@ -23,7 +23,7 @@ process.env.SNOWPACK_PUBLIC_PATH = process.env.CDN_VERSION
     ? ''
     : process.env.PUBLIC_PATH;
 
-// BASE_URI is for env and router, must be local address which starts with a `/` or empty string
+// BASE_URI is for env, router and workers. Must be local address which starts with a `/` or empty string
 // if it is not set and PUBLIC_PATH is not a CDN address, it will be set to the same value of PUBLIC_PATH
 process.env.SNOWPACK_PUBLIC_BASE_URI =
     process.env.SNOWPACK_PUBLIC_PATH.startsWith('/') || process.env.PUBLIC_PATH === ''

frontend/packages/core/package.json

 {
     "name": "@visualdl/core",
-    "version": "2.1.1",
+    "version": "2.1.4",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
         "visualdl",
@@ -35,8 +35,8 @@
     ],
     "dependencies": {
         "@tippyjs/react": "4.2.0",
-        "@visualdl/netron": "2.1.1",
-        "@visualdl/wasm": "2.1.1",
+        "@visualdl/netron": "2.1.4",
+        "@visualdl/wasm": "2.1.4",
         "bignumber.js": "9.0.1",
         "d3": "6.3.1",
         "d3-format": "2.0.0",
@@ -105,7 +105,7 @@
         "@types/react-router-dom": "5.1.6",
         "@types/snowpack-env": "2.3.3",
         "@types/styled-components": "5.1.7",
-        "@visualdl/mock": "2.1.1",
+        "@visualdl/mock": "2.1.4",
         "babel-plugin-styled-components": "1.12.0",
         "dotenv": "8.2.0",
         "enhanced-resolve": "5.4.1",

frontend/packages/core/src/hooks/useWorker.ts

@@ -19,7 +19,7 @@ import {useEffect, useState} from 'react';
 import type {InitializeData} from '~/worker';
 import {WebWorker} from '~/worker';
 
-const PUBLIC_PATH: string = import.meta.env.SNOWPACK_PUBLIC_PATH;
+const BASE_URI: string = import.meta.env.SNOWPACK_PUBLIC_BASE_URI;
 
 type WorkerResult<D, E extends Error> = {
     data?: D;
@@ -31,7 +31,7 @@ const useWorker = <D, P = unknown, E extends Error = Error>(name: string, params
     const [result, setResult] = useState<WorkerResult<D, E>>({});
 
     useEffect(() => {
-        const worker = new WebWorker(`${PUBLIC_PATH}/_dist_/worker/${name}.js`, {type: 'module'});
+        const worker = new WebWorker(`${BASE_URI}/_dist_/worker/${name}.js`, {type: 'module'});
         worker.emit<InitializeData>('INITIALIZE', {env: import.meta.env});
         worker.on('INITIALIZED', () => {
             setResult({worker});

frontend/packages/demo/builder/high-dimensional.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import type {Worker} from './types';
+import type {Embedding, Worker} from './types';
 
 const worker: Worker = async io => {
     const components = await io.getData<string[]>('/components');
@@ -22,7 +22,12 @@ const worker: Worker = async io => {
         return;
     }
 
-    // await io.save<Record<string, string[]>>('/embedding/embedding');
+    const list = await io.save<Embedding[]>('/embedding/list');
+    await Promise.all(
+        list.map(({name}) =>
+            Promise.all([io.saveBinary('/embedding/tensor', {name}), io.saveBinary('/embedding/metadata', {name})])
+        )
+    );
 };
 
 export default worker;

frontend/packages/demo/builder/index.ts

@@ -17,7 +17,6 @@
 /* eslint-disable no-console */
 
 import IO from './io';
-import {SIGINT} from 'constants';
 import type {Worker} from './types';
 import getPort from 'get-port';
 import mkdirp from 'mkdirp';
@@ -63,7 +62,7 @@ async function start() {
 
     const stop = () => {
         if (!p.killed) {
-            p.kill(SIGINT);
+            p.kill('SIGINT');
         }
     };
 

frontend/packages/demo/builder/io.ts

@@ -174,7 +174,7 @@ export default class IO {
 
         const response = await this.fetch(uri, query);
         if (!response.ok) {
-            throw new Error('not ok');
+            throw new Error(`not ok: ${uri}`);
         }
 
         let content: ResponseData<T> | ArrayBuffer;

frontend/packages/demo/builder/scalar.ts

@@ -16,6 +16,8 @@
 
 import type {Data, Worker} from './types';
 
+const DataTypes = ['csv', 'tsv'];
+
 const worker: Worker = async io => {
     const components = await io.getData<string[]>('/components');
     if (!components.includes('scalar')) {
@@ -26,7 +28,10 @@ const worker: Worker = async io => {
     const q = [];
     for (const [index, run] of runs.entries()) {
         for (const tag of tags[index]) {
-            q.push(io.save('/scalar/list', {run, tag}));
+            q.push(
+                io.save('/scalar/list', {run, tag}),
+                ...DataTypes.map(type => io.saveBinary('/scalar/data', {run, tag, type}))
+            );
         }
     }
     await Promise.all(q);

frontend/packages/demo/builder/types.ts

@@ -22,3 +22,9 @@ export type Data = {
     runs: string[];
     tags: string[][];
 };
+
+export type Embedding = {
+    name: string;
+    shape: [number, number];
+    path: string;
+};

frontend/packages/demo/package.json

 {
     "name": "@visualdl/demo",
-    "version": "2.1.1",
+    "version": "2.1.4",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
         "visualdl",

frontend/packages/mock/package.json

 {
     "name": "@visualdl/mock",
-    "version": "2.1.1",
+    "version": "2.1.4",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
         "visualdl",

frontend/packages/netron/package.json

 {
     "name": "@visualdl/netron",
-    "version": "2.1.1",
+    "version": "2.1.4",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
         "visualdl",

frontend/packages/server/package.json

 {
     "name": "@visualdl/server",
-    "version": "2.1.1",
+    "version": "2.1.4",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
         "visualdl",
@@ -25,6 +25,7 @@
     "scripts": {
         "build": "tsc",
         "dev": "cross-env NODE_ENV=development nodemon --watch index.ts --exec \"ts-node index.ts\"",
+        "dev:demo": "cross-env DEMO=1 yarn dev",
         "start": "pm2-runtime ecosystem.config.js",
         "test": "echo \"Error: no test specified\" && exit 0"
     },
@@ -36,7 +37,7 @@
         "ecosystem.config.d.ts"
     ],
     "dependencies": {
-        "@visualdl/core": "2.1.1",
+        "@visualdl/core": "2.1.4",
         "dotenv": "8.2.0",
         "enhanced-resolve": "5.4.1",
         "express": "4.17.1",
@@ -47,14 +48,14 @@
         "@types/enhanced-resolve": "3.0.6",
         "@types/express": "4.17.9",
         "@types/node": "14.14.16",
-        "@visualdl/mock": "2.1.1",
+        "@visualdl/mock": "2.1.4",
         "cross-env": "7.0.3",
         "nodemon": "2.0.6",
         "ts-node": "9.1.1",
         "typescript": "4.0.5"
     },
     "optionalDependencies": {
-        "@visualdl/demo": "2.1.1"
+        "@visualdl/demo": "2.1.4"
     },
     "engines": {
         "node": ">=12",

frontend/packages/wasm/package.json

 {
     "name": "@visualdl/wasm",
-    "version": "2.1.1",
+    "version": "2.1.4",
     "title": "VisualDL",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [

frontend/scripts/build.sh

@@ -11,6 +11,7 @@ OUTPUT="output"
 OUTPUT_PATH="$WORKING_PATH/$OUTPUT"
 
 if [ -f "$HOME/.cargo/env" ]; then
+    # shellcheck source=/dev/null
     source "$HOME/.cargo/env"
 fi
 

frontend/scripts/install.sh

@@ -4,7 +4,7 @@ set -e
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-cd $SCRIPT_DIR/..
+cd "${SCRIPT_DIR}/.."
 
 # rust toolchain
 # https://rustup.rs/

frontend/scripts/lint-staged.sh

@@ -2,10 +2,8 @@
 
 set -e
 
-echo $0
-
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-cd $SCRIPT_DIR/..
+cd "${SCRIPT_DIR}/.."
 
 ./node_modules/.bin/lint-staged --no-stash

frontend/scripts/publish.sh

+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+PACKAGES="${SCRIPT_DIR}/../packages"
+
+for dirname in "${PACKAGES}"/*; do
+    package="${PACKAGES}/${dirname}"
+    if [ -d "$package" ]; then
+        (cd "$package" && npm publish)
+    fi
+done

requirements.txt

-numpy
-requests
+bce-python-sdk
 flake8 >= 3.7.9
-Pillow >= 7.0.0
-pre-commit
 flask >= 1.1.1
 Flask-Babel >= 1.0.0
-six >= 1.14.0
+numpy
+Pillow >= 7.0.0
+pre-commit
 protobuf >= 3.11.0
-bce-python-sdk
+requests
+shellcheck-py
+six >= 1.14.0

scripts/build.sh

@@ -11,6 +11,7 @@ build_frontend() {
     mkdir -p "$FRONTEND_DIST"
 
     cd "$FRONTEND_DIR"
+    # shellcheck disable=SC1091
     . ./scripts/install.sh
     SCOPE="serverless" \
       PUBLIC_PATH="{{PUBLIC_PATH}}" \

scripts/setup.sh

@@ -3,22 +3,20 @@ set -e
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-cd $SCRIPT_DIR/..
+cd "${SCRIPT_DIR}/.."
 
-which python > /dev/null 2>&1
-if [ $? -ne 0 ]; then
+if which python > /dev/null 2>&1; then
     echo "You need to install Python 3.5+"
     exit 1
 fi
 python --version
 
-which node > /dev/null 2>&1
-if [ $? -ne 0 ]; then
+if which node > /dev/null 2>&1; then
     echo "You need to install nodejs 14+"
     exit 1
 fi
 node --version
 
-pip install pre-commit && pre-commit install
 pip install --disable-pip-version-check -r requirements.txt
 (cd frontend && scripts/install.sh)
+pre-commit install