Demo (#737)

* rewrite logger

* feat: add demo

* bump frontend 2.0.0-beta.50

* fix: nodejs 12.x compatibility

* chore: update dependencies

frontend/.eslintrc.js

@@ -16,7 +16,13 @@ module.exports = {
     },
     overrides: [
         {
-            files: ['packages/cli/**/*', 'packages/mock/**/*', 'packages/server/**/*', 'packages/serverless/**/*'],
+            files: [
+                'packages/cli/**/*',
+                'packages/mock/**/*',
+                'packages/demo/**/*',
+                'packages/server/**/*',
+                'packages/serverless/**/*'
+            ],
             extends: [
                 'plugin:@typescript-eslint/recommended',
                 'prettier/@typescript-eslint',

frontend/lerna.json

@@ -2,7 +2,7 @@
     "packages": [
         "packages/*"
     ],
-    "version": "2.0.0-beta.49",
+    "version": "2.0.0-beta.50",
     "npmClient": "yarn",
     "useWorkspaces": true,
     "command": {

frontend/package.json

@@ -38,12 +38,12 @@
         "version": "yarn format && git add -A"
     },
     "devDependencies": {
-        "@typescript-eslint/eslint-plugin": "3.7.1",
-        "@typescript-eslint/parser": "3.7.1",
-        "eslint": "7.5.0",
+        "@typescript-eslint/eslint-plugin": "3.8.0",
+        "@typescript-eslint/parser": "3.8.0",
+        "eslint": "7.6.0",
         "eslint-config-prettier": "6.11.0",
         "eslint-plugin-prettier": "3.1.4",
-        "eslint-plugin-react": "7.20.4",
+        "eslint-plugin-react": "7.20.5",
         "eslint-plugin-react-hooks": "4.0.8",
         "husky": "4.2.5",
         "lerna": "3.22.1",

frontend/packages/cli/index.ts

@@ -4,6 +4,7 @@
 /* eslint-disable no-console */
 
 import ora, {Ora} from 'ora';
+
 import ecosystem from '@visualdl/server/ecosystem.config';
 import pm2 from 'pm2';
 
@@ -27,6 +28,8 @@ const argv = require('yargs')
     .nargs('b', 1)
     .nargs('backend', 1)
     .describe('b', 'Backend API address')
+    .boolean('demo')
+    .describe('demo', 'Run in demo mode')
     .boolean('open')
     .describe('open', 'Open browser when server is ready')
     .help('h')
@@ -92,7 +95,8 @@ pm2.connect(err => {
                         ...app.env,
                         HOST: host,
                         PORT: port + '',
-                        BACKEND: argv.backend
+                        BACKEND: argv.backend,
+                        DEMO: argv.demo ? '1' : ''
                     }
                 },
                 err => {

frontend/packages/cli/package.json

 {
     "name": "@visualdl/cli",
-    "version": "2.0.0-beta.49",
+    "version": "2.0.0-beta.50",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
         "visualdl",
@@ -34,14 +34,14 @@
         "dist"
     ],
     "dependencies": {
-        "@visualdl/server": "2.0.0-beta.49",
+        "@visualdl/server": "2.0.0-beta.50",
         "open": "7.1.0",
         "ora": "4.0.5",
         "pm2": "4.4.0",
         "yargs": "15.4.1"
     },
     "devDependencies": {
-        "@types/node": "14.0.26",
+        "@types/node": "14.0.27",
         "@types/yargs": "15.0.5",
         "cross-env": "7.0.2",
         "ts-node": "8.10.2",

frontend/packages/core/components/RangeSlider.tsx

@@ -11,8 +11,7 @@ import {
     primaryFocusedColor,
     sameBorder,
     size,
-    textLighterColor,
-    transitionProps
+    textLighterColor
 } from '~/utils/style';
 
 import styled from 'styled-components';

frontend/packages/core/package.json

 {
     "name": "@visualdl/core",
-    "version": "2.0.0-beta.49",
+    "version": "2.0.0-beta.50",
     "title": "VisualDL",
     "description": "A platform to visualize the deep learning process and result.",
     "keywords": [
@@ -34,9 +34,9 @@
     },
     "dependencies": {
         "@tippyjs/react": "4.1.0",
-        "@visualdl/i18n": "2.0.0-beta.49",
-        "@visualdl/netron": "2.0.0-beta.49",
-        "@visualdl/wasm": "2.0.0-beta.49",
+        "@visualdl/i18n": "2.0.0-beta.50",
+        "@visualdl/netron": "2.0.0-beta.50",
+        "@visualdl/wasm": "2.0.0-beta.50",
         "bignumber.js": "9.0.0",
         "d3-format": "1.4.4",
         "echarts": "4.8.0",
@@ -61,38 +61,38 @@
         "react-toastify": "6.0.8",
         "save-svg-as-png": "1.4.17",
         "styled-components": "5.1.1",
-        "swr": "0.2.3",
-        "tippy.js": "6.2.5"
+        "swr": "0.3.0",
+        "tippy.js": "6.2.6"
     },
     "devDependencies": {
-        "@babel/core": "7.10.5",
+        "@babel/core": "7.11.1",
         "@types/d3-format": "1.3.1",
         "@types/echarts": "4.6.4",
         "@types/enzyme": "3.10.5",
         "@types/enzyme-adapter-react-16": "1.0.6",
         "@types/file-saver": "2.0.1",
-        "@types/jest": "26.0.7",
+        "@types/jest": "26.0.8",
         "@types/lodash": "4.14.158",
         "@types/mime-types": "2.1.0",
-        "@types/node": "14.0.26",
+        "@types/node": "14.0.27",
         "@types/nprogress": "0.2.0",
-        "@types/react": "16.9.43",
+        "@types/react": "16.9.44",
         "@types/react-dom": "16.9.8",
         "@types/react-rangeslider": "2.2.3",
-        "@types/styled-components": "5.1.1",
-        "@visualdl/mock": "2.0.0-beta.49",
+        "@types/styled-components": "5.1.2",
+        "@visualdl/mock": "2.0.0-beta.50",
         "babel-plugin-emotion": "10.0.33",
-        "babel-plugin-styled-components": "1.10.7",
+        "babel-plugin-styled-components": "1.11.1",
         "babel-plugin-typescript-to-proptypes": "1.4.0",
         "copy-webpack-plugin": "6.0.3",
         "core-js": "3.6.5",
         "cross-env": "7.0.2",
-        "css-loader": "4.0.0",
+        "css-loader": "4.2.0",
         "enhanced-resolve": "4.3.0",
         "enzyme": "3.11.0",
         "enzyme-adapter-react-16": "1.15.2",
         "enzyme-to-json": "3.5.0",
-        "jest": "26.1.0",
+        "jest": "26.2.2",
         "ora": "4.0.5",
         "ts-jest": "26.1.4",
         "typescript": "3.9.7",

frontend/packages/demo/.gitignore

+data

frontend/packages/demo/builder/audio.ts

+import type {Worker} from './types';
+
+interface Audio {
+    step: number;
+    wallTime: number;
+}
+
+const worker: Worker = async io => {
+    const components = await io.getData<string[]>('/components');
+    if (!components.includes('audio')) {
+        return;
+    }
+
+    const tagsMap = await io.save<Record<string, string[]>>('/audio/tags');
+    for (const [run, tags] of Object.entries(tagsMap)) {
+        for (const tag of tags) {
+            const list = await io.save<Audio[]>('/audio/list', {run, tag});
+            for (const [index, audio] of Object.entries(list)) {
+                await io.saveBinary('/audio/audio', {run, tag, index, ts: audio.wallTime});
+            }
+        }
+    }
+};
+
+export default worker;

frontend/packages/demo/builder/common.ts

+import type {Worker} from './types';
+
+const worker: Worker = async io => {
+    await io.save<string[]>('/components');
+    await io.save<string[]>('/runs');
+};
+
+export default worker;

frontend/packages/demo/builder/graph.ts

+import type {Worker} from './types';
+
+const worker: Worker = async io => {
+    await io.saveBinary('/graph/graph');
+};
+
+export default worker;

frontend/packages/demo/builder/high-dimensional.ts

+import type {Worker} from './types';
+
+const worker: Worker = async io => {
+    const components = await io.getData<string[]>('/components');
+    if (!components.includes('embeddings')) {
+        return;
+    }
+
+    // await io.save<Record<string, string[]>>('/embedding/embedding');
+};
+
+export default worker;

frontend/packages/demo/builder/histogram.ts

+import type {Worker} from './types';
+
+const worker: Worker = async io => {
+    const components = await io.getData<string[]>('/components');
+    if (!components.includes('histogram')) {
+        return;
+    }
+
+    const tagsMap = await io.save<Record<string, string[]>>('/histogram/tags');
+    const q = [];
+    for (const [run, tags] of Object.entries(tagsMap)) {
+        for (const tag of tags) {
+            q.push(io.save('/histogram/list', {run, tag}));
+        }
+    }
+    await Promise.all(q);
+};
+
+export default worker;

frontend/packages/demo/builder/image.ts

+import type {Worker} from './types';
+
+interface Image {
+    step: number;
+    wallTime: number;
+}
+
+const worker: Worker = async io => {
+    const components = await io.getData<string[]>('/components');
+    if (!components.includes('image')) {
+        return;
+    }
+
+    const tagsMap = await io.save<Record<string, string[]>>('/image/tags');
+    for (const [run, tags] of Object.entries(tagsMap)) {
+        for (const tag of tags) {
+            const list = await io.save<Image[]>('/image/list', {run, tag});
+            for (const [index, image] of Object.entries(list)) {
+                await io.saveBinary('/image/image', {run, tag, index, ts: image.wallTime});
+            }
+        }
+    }
+};
+
+export default worker;

frontend/packages/demo/builder/index.ts

+/* eslint-disable no-console */
+
+import IO from './io';
+import {SIGINT} from 'constants';
+import type {Worker} from './types';
+import getPort from 'get-port';
+import mkdirp from 'mkdirp';
+import path from 'path';
+import rimraf from 'rimraf';
+import {spawn} from 'child_process';
+
+const host = '127.0.0.1';
+const publicPath = '/visualdl';
+const pages = ['common', 'scalar', 'histogram', 'image', 'audio', 'graph', 'pr-curve', 'high-dimensional'];
+const dataDir = path.resolve(__dirname, '../data');
+
+async function start() {
+    rimraf.sync(dataDir);
+
+    const port = await getPort({host});
+
+    mkdirp.sync(dataDir);
+
+    const io = new IO(`http://${host}:${port}${publicPath}`, dataDir);
+
+    const p = spawn(
+        'visualdl',
+        [
+            '--logdir',
+            '.',
+            '--model',
+            './graph/__model__',
+            '--host',
+            host,
+            '--port',
+            String(port),
+            '--public-path',
+            publicPath
+        ],
+        {
+            cwd: path.resolve(__dirname, '../logs'),
+            stdio: ['ignore', 'pipe', 'pipe']
+        }
+    );
+
+    p.on('error', err => console.error(err));
+
+    const stop = () => {
+        if (!p.killed) {
+            p.kill(SIGINT);
+        }
+    };
+
+    const check = async (data: Buffer) => {
+        const message = data.toString();
+        if (message.startsWith('Running VisualDL')) {
+            p.stdout.off('data', check);
+            p.stderr.off('data', check);
+            await Promise.all(
+                pages.map(
+                    page =>
+                        new Promise((resolve, reject) => {
+                            import(`./${page}`)
+                                .then(data => data.default)
+                                .then((worker: Worker) => worker(io).then(resolve))
+                                .catch(reject);
+                        })
+                )
+            );
+            await io.generateMeta();
+            stop();
+        }
+    };
+
+    p.stdout.on('data', check);
+    p.stderr.on('data', check);
+
+    process.on('exit', stop);
+}
+
+start();

frontend/packages/demo/builder/io.ts

+/* eslint-disable no-console */
+
+import crypto, {BinaryLike} from 'crypto';
+
+import fetch from 'node-fetch';
+import {promises as fs} from 'fs';
+import mime from 'mime-types';
+import mkdirp from 'mkdirp';
+import path from 'path';
+import querystring from 'querystring';
+
+const apiUrl = '/api';
+
+type Query = Record<string, string | number> | null;
+
+interface WriteOptions {
+    type?: 'json' | 'buffer';
+}
+
+interface MetaData {
+    uri: string;
+    query?: Record<string, string | string[]>;
+    filename: string;
+    headers: Record<string, string>;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+interface ResponseData<T = any> {
+    status: number;
+    msg?: string;
+    data: T;
+}
+
+function isEmpty(obj: Record<string, unknown> | null | undefined) {
+    if (obj == null) {
+        return true;
+    }
+    return !Object.keys(obj).length;
+}
+
+export default class IO {
+    public static readonly metaFileName = 'meta.json';
+    public static readonly dataPath = 'data';
+    public static readonly hashFunction = 'md4';
+
+    protected readonly url: string;
+    protected readonly dataDir: string;
+
+    protected metadata: MetaData[] = [];
+
+    constructor(url: string, dataDir: string) {
+        this.url = url;
+        this.dataDir = dataDir;
+    }
+
+    public static isSameUri(url1: Pick<MetaData, 'uri' | 'query'>, url2: Pick<MetaData, 'uri' | 'query'>) {
+        if (url1.uri !== url2.uri) {
+            return false;
+        }
+        if (!isEmpty(url2.query)) {
+            if (isEmpty(url1.query)) {
+                return false;
+            }
+            for (const [key, value] of Object.entries(url2.query)) {
+                const existValue = url1.query[key];
+                if (existValue !== value) {
+                    if (Array.isArray(value) && Array.isArray(existValue)) {
+                        const count = value.reduce<Record<string, number>>((m, v) => {
+                            if (m[v] == null) {
+                                m[v] = 1;
+                            } else {
+                                m[v]++;
+                            }
+                            return m;
+                        }, {});
+                        for (const i of existValue) {
+                            if (count[i] == null) {
+                                return false;
+                            }
+                            count[i]--;
+                        }
+                        return Object.values(count).every(c => c === 0);
+                    }
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            return isEmpty(url1.query);
+        }
+    }
+
+    private generateFilename(content: BinaryLike) {
+        const hash = crypto.createHash(IO.hashFunction);
+        hash.update(content);
+        return hash.digest('hex');
+    }
+
+    private addMeta(meta: MetaData) {
+        const exist = this.metadata.find(data => IO.isSameUri(data, meta));
+        if (!exist) {
+            this.metadata.push(meta);
+        }
+    }
+
+    protected async write(
+        filePath: string,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        content: Record<string, any> | Buffer,
+        contentType: string,
+        options?: WriteOptions | WriteOptions['type']
+    ) {
+        const type = 'string' === typeof options ? options : options?.type ?? 'json';
+
+        const fileDir = path.join(this.dataDir, IO.dataPath, filePath);
+        await mkdirp(fileDir);
+        let fileContent: Buffer;
+        let extname: string;
+        if (type === 'buffer') {
+            extname = mime.extension(contentType) || '';
+            if (extname) {
+                extname = '.' + extname;
+            }
+            fileContent = content as Buffer;
+        } else {
+            extname = '.json';
+            fileContent = Buffer.from(JSON.stringify(content), 'utf-8');
+        }
+
+        const filename = this.generateFilename(fileContent) + extname;
+        await fs.writeFile(path.join(fileDir, filename), fileContent, {
+            encoding: null,
+            flag: 'w'
+        });
+        console.log(`write file ${path.join(filePath, filename)}`);
+        return filename;
+    }
+
+    fetch(uri: string, query?: Query) {
+        let url = this.url + apiUrl + uri;
+        if (!isEmpty(query)) {
+            url += '?' + querystring.stringify(query);
+        }
+        try {
+            return fetch(url);
+        } catch (e) {
+            console.error(e);
+        }
+    }
+
+    protected async fetchAndWrite<T>(uri: string, query?: Query, options?: WriteOptions | WriteOptions['type']) {
+        const type = 'string' === typeof options ? options : options?.type ?? 'json';
+
+        const response = await this.fetch(uri, query);
+        if (!response.ok) {
+            throw new Error('not ok');
+        }
+
+        let content: ResponseData<T> | ArrayBuffer;
+        if (type === 'buffer') {
+            content = await response.buffer();
+        } else {
+            content = (await response.json()) as ResponseData<T>;
+        }
+        const filename = await this.write(uri, content, response.headers.get('content-type'), options);
+        this.addMeta({
+            uri,
+            query: isEmpty(query) ? undefined : querystring.parse(querystring.stringify(query)),
+            filename,
+            headers: ['Content-Type', 'Content-Disposition'].reduce((m, t) => {
+                m[t] = response.headers.get(t) || undefined;
+                return m;
+            }, {})
+        });
+        return content;
+    }
+
+    async save<T>(uri: string, query?: Query) {
+        return ((await this.fetchAndWrite<T>(uri, query, 'json')) as ResponseData<T>).data;
+    }
+
+    async saveBinary(uri: string, query?: Query) {
+        return (await this.fetchAndWrite(uri, query, 'buffer')) as Buffer;
+    }
+
+    async getData<T>(uri: string, query?: Query) {
+        return ((await (await this.fetch(uri, query)).json()) as ResponseData<T>).data;
+    }
+
+    generateMeta() {
+        return fs.writeFile(path.join(this.dataDir, IO.metaFileName), JSON.stringify(this.metadata), {
+            encoding: 'utf-8',
+            flag: 'w'
+        });
+    }
+
+    sleep(time: number) {
+        return new Promise(resolve => {
+            setTimeout(resolve, time);
+        });
+    }
+}

frontend/packages/demo/builder/pr-curve.ts

+import type {Worker} from './types';
+
+const worker: Worker = async io => {
+    const components = await io.getData<string[]>('/components');
+    if (!components.includes('pr_curve')) {
+        return;
+    }
+
+    const tagsMap = await io.save<Record<string, string[]>>('/pr-curve/tags');
+    for (const [run, tags] of Object.entries(tagsMap)) {
+        await io.save('/pr-curve/steps', {run});
+        for (const tag of tags) {
+            await io.save('/pr-curve/list', {run, tag});
+        }
+    }
+};
+
+export default worker;

frontend/packages/demo/builder/scalar.ts

+import type {Worker} from './types';
+
+const worker: Worker = async io => {
+    const components = await io.getData<string[]>('/components');
+    if (!components.includes('scalar')) {
+        return;
+    }
+
+    const tagsMap = await io.save<Record<string, string[]>>('/scalar/tags');
+    const q = [];
+    for (const [run, tags] of Object.entries(tagsMap)) {
+        for (const tag of tags) {
+            q.push(io.save('/scalar/list', {run, tag}));
+        }
+    }
+    await Promise.all(q);
+};
+
+export default worker;

frontend/packages/demo/builder/types.ts

+import type IO from './io';
+
+export type Worker = (io: IO) => Promise<void>;

frontend/packages/demo/index.ts

+import {Request, Response} from 'express';
+
+import IO from './builder/io';
+import meta from './data/meta.json';
+import path from 'path';
+
+function notFound(res: Response) {
+    res.status(404).send({
+        status: 1,
+        msg: 'Not found'
+    });
+}
+
+export default async (req: Request, res: Response) => {
+    const method = req.path;
+
+    if (!method) {
+        return notFound(res);
+    }
+
+    const data = meta.find(item =>
+        IO.isSameUri(item, {uri: method, query: req.query as Record<string, string | string[]>})
+    );
+
+    if (!data) {
+        return notFound(res);
+    }
+
+    res.sendFile(path.join(__dirname, 'data/data', data.uri, data.filename), {
+        headers: data.headers
+    });
+};