rewrite roc curve (#881)

* 增加 roc 曲线

所有更改参照pr曲线更改，在公式计算的地方更改为了 roc 的计算公式

* Update index.ts

语法改正

* Update reducers.ts

修改语法错误

* 生成record_pb2.py

编译生成record_pb2.py
修改 record.proto 序号问题

* Update base_component.py

更正参数

* Update api.py

修正语法错误

* add data/roc-curve

前端页面

* add resource

add resource

* Update lib.py

* docs: remove luckydraw

* feat: download raw data

* fix conflict

* chore: rewrite pr-curve & roc-curve

* fix: unexpected background color in dark mode
Co-authored-by: Niceriver97 <1105107356@qq.com>

demo/components/roc_curve_test.py

+# Copyright (c) 2020 VisualDL Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =======================================================================
+# coding=utf-8
+from visualdl import LogWriter
+import numpy as np
+
+with LogWriter("./log/roc_curve_test/train") as writer:
+    for step in range(3):
+        labels = np.random.randint(2, size=100)
+        predictions = np.random.rand(100)
+        writer.add_roc_curve(tag='roc_curve',
+                            labels=labels,
+                            predictions=predictions,
+                            step=step,
+                            num_thresholds=5)

frontend/packages/core/package.json

@@ -47,6 +47,7 @@
         "i18next": "19.8.4",
         "i18next-browser-languagedetector": "6.0.1",
         "i18next-fetch-backend": "3.0.0",
+        "jszip": "3.5.0",
         "lodash": "4.17.20",
         "mime-types": "2.1.27",
         "moment": "2.29.1",

frontend/packages/core/public/locales/en/common.json

@@ -12,8 +12,9 @@
     "image": "Image",
     "loading": "Please wait while loading data",
     "next-page": "Next Page",
-    "pr-curve": "PR-Curve",
+    "pr-curve": "PR Curve",
     "previous-page": "Prev Page",
+    "roc-curve": "ROC Curve",
     "run": "Run",
     "running": "Running",
     "runs": "Runs",

frontend/packages/core/public/locales/en/scalar.json

 {
+    "download-data": "Download data",
     "download-image": "Download image",
     "ignore-outliers": "Ignore outliers in chart scaling",
     "max": "Max.",

frontend/packages/core/public/locales/zh/common.json

@@ -13,6 +13,7 @@
     "loading": "数据载入中，请稍等",
     "next-page": "下一页",
     "pr-curve": "PR曲线",
+    "roc-curve": "ROC曲线",
     "previous-page": "上一页",
     "run": "运行",
     "running": "运行中",

frontend/packages/core/public/locales/zh/scalar.json

 {
+    "download-data": "下载数据",
     "download-image": "下载图片",
     "ignore-outliers": "图表缩放时忽略极端值",
     "max": "最大值",

frontend/packages/core/snowpack.config.js

@@ -71,7 +71,6 @@ module.exports = {
         clean: true
     },
     installOptions: {
-        polyfillNode: true,
-        namedExports: ['file-saver']
+        polyfillNode: true
     }
 };

frontend/packages/core/src/components/ChartToolbox.tsx

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import React, {FunctionComponent, useCallback, useState} from 'react';
+import React, {FunctionComponent, useCallback, useContext, useMemo, useState} from 'react';
 import {WithStyled, em, rem, transitionProps} from '~/utils/style';
 
 import Icon from '~/components/Icon';
@@ -48,6 +48,28 @@ const ToolboxItem = styled.a<{active?: boolean}>`
     }
 `;
 
+const ChartToolboxMenu = styled.div`
+    background-color: var(--background-color);
+    ${transitionProps('background-color')};
+
+    > a {
+        cursor: pointer;
+        display: block;
+        padding: ${rem(10)};
+        background-color: var(--background-color);
+        ${transitionProps(['color', 'background-color'])};
+
+        &:hover {
+            background-color: var(--background-focused-color);
+        }
+    }
+`;
+
+interface ChartToolboxItemChild {
+    label: string;
+    onClick?: () => unknown;
+}
+
 type BaseChartToolboxItem = {
     icon: Icons;
     tooltip?: string;
@@ -65,12 +87,111 @@ type ToggleChartToolboxItem = {
     onClick?: (value: boolean) => unknown;
 } & BaseChartToolboxItem;
 
-export type ChartToolboxItem = NormalChartToolboxItem | ToggleChartToolboxItem;
+type MenuChartToolboxItem = {
+    toggle?: false;
+    tooltip: undefined;
+    menuList: ChartToolboxItemChild[];
+} & BaseChartToolboxItem;
+
+export type ChartToolboxItem = NormalChartToolboxItem | ToggleChartToolboxItem | MenuChartToolboxItem;
+
+const ChartToolboxIcon = React.forwardRef<
+    HTMLAnchorElement,
+    {
+        toggle?: boolean;
+        icon: Icons;
+        activeIcon?: Icons;
+        activeStatus?: boolean;
+        onClick?: () => unknown;
+    }
+>(({toggle, icon, activeIcon, activeStatus, onClick}, ref) => {
+    return (
+        <ToolboxItem ref={ref} active={toggle && !activeIcon && activeStatus} onClick={() => onClick?.()}>
+            <Icon type={toggle ? (activeStatus && activeIcon) || icon : icon} />
+        </ToolboxItem>
+    );
+});
+
+ChartToolboxIcon.displayName = 'ChartToolboxIcon';
+
+type ChartToolboxItemProps = {
+    tooltipPlacement?: 'top' | 'bottom' | 'left' | 'right';
+};
 
 type ChartToolboxProps = {
     items: ChartToolboxItem[];
     reversed?: boolean;
-    tooltipPlacement?: 'top' | 'bottom' | 'left' | 'right';
+} & ChartToolboxItemProps;
+
+const ChartToolboxItemContext = React.createContext<ChartToolboxItemProps>({
+    tooltipPlacement: 'top'
+});
+
+const NormalChartToolbox: FunctionComponent<NormalChartToolboxItem> = ({icon, tooltip, onClick}) => {
+    const toolboxIcon = useMemo(() => <ChartToolboxIcon icon={icon} onClick={onClick} />, [icon, onClick]);
+    const {tooltipPlacement} = useContext(ChartToolboxItemContext);
+
+    return tooltip ? (
+        <Tippy content={tooltip} placement={tooltipPlacement || 'top'} theme="tooltip">
+            {toolboxIcon}
+        </Tippy>
+    ) : (
+        toolboxIcon
+    );
+};
+
+const ToggleChartToolbox: FunctionComponent<ToggleChartToolboxItem> = ({
+    icon,
+    tooltip,
+    activeIcon,
+    activeTooltip,
+    onClick
+}) => {
+    const [active, setActive] = useState(false);
+    const click = useCallback(() => {
+        setActive(a => {
+            onClick?.(!a);
+            return !a;
+        });
+    }, [onClick]);
+    const toolboxIcon = useMemo(
+        () => <ChartToolboxIcon icon={icon} activeIcon={activeIcon} activeStatus={active} toggle onClick={click} />,
+        [icon, activeIcon, active, click]
+    );
+    const {tooltipPlacement} = useContext(ChartToolboxItemContext);
+
+    return tooltip ? (
+        <Tippy content={(active && activeTooltip) || tooltip} placement={tooltipPlacement || 'top'} theme="tooltip">
+            {toolboxIcon}
+        </Tippy>
+    ) : (
+        toolboxIcon
+    );
+};
+
+const MenuChartToolbox: FunctionComponent<MenuChartToolboxItem> = ({icon, menuList}) => {
+    return (
+        <Tippy
+            content={
+                <ChartToolboxMenu>
+                    {menuList.map((item, index) => (
+                        <a key={index} onClick={() => item.onClick?.()}>
+                            {item.label}
+                        </a>
+                    ))}
+                </ChartToolboxMenu>
+            }
+            placement="right-start"
+            animation="shift-away-subtle"
+            interactive
+            hideOnClick={false}
+            arrow={false}
+            role="menu"
+            theme="menu"
+        >
+            <ChartToolboxIcon icon={icon} />
+        </Tippy>
+    );
 };
 
 const ChartToolbox: FunctionComponent<ChartToolboxProps & WithStyled> = ({
@@ -79,56 +200,25 @@ const ChartToolbox: FunctionComponent<ChartToolboxProps & WithStyled> = ({
     reversed,
     className
 }) => {
-    const [activeStatus, setActiveStatus] = useState<boolean[]>(new Array(items.length).fill(false));
-    const onClick = useCallback(
-        (index: number) => {
-            const item = items[index];
-            if (item.toggle) {
-                item.onClick?.(!activeStatus[index]);
-                setActiveStatus(m => {
-                    const n = [...m];
-                    n.splice(index, 1, !m[index]);
-                    return n;
-                });
-            } else {
-                item.onClick?.();
-            }
-        },
-        [items, activeStatus]
-    );
-
-    const getToolboxItem = useCallback(
-        (item: ChartToolboxItem, index: number) => (
-            <ToolboxItem
-                key={index}
-                active={item.toggle && !item.activeIcon && activeStatus[index]}
-                onClick={() => onClick(index)}
-            >
-                <Icon type={item.toggle ? (activeStatus[index] && item.activeIcon) || item.icon : item.icon} />
-            </ToolboxItem>
-        ),
-        [activeStatus, onClick]
-    );
+    const contextValue = useMemo(() => ({tooltipPlacement}), [tooltipPlacement]);
 
     return (
         <>
             <Toolbox className={className} size={items.length} reversed={reversed}>
-                {items.map((item, index) =>
-                    item.tooltip ? (
-                        <Tippy
-                            content={
-                                item.toggle ? (activeStatus[index] && item.activeTooltip) || item.tooltip : item.tooltip
-                            }
-                            placement={tooltipPlacement || 'top'}
-                            theme="tooltip"
-                            key={index}
-                        >
-                            {getToolboxItem(item, index)}
-                        </Tippy>
-                    ) : (
-                        getToolboxItem(item, index)
-                    )
-                )}
+                <ChartToolboxItemContext.Provider value={contextValue}>
+                    {items.map((item, index) => {
+                        if ((item as MenuChartToolboxItem).menuList) {
+                            const i = item as MenuChartToolboxItem;
+                            return <MenuChartToolbox {...i} key={index} />;
+                        }
+                        if ((item as ToggleChartToolboxItem).toggle) {
+                            const i = item as ToggleChartToolboxItem;
+                            return <ToggleChartToolbox {...i} key={index} />;
+                        }
+                        const i = item as NormalChartToolboxItem;
+                        return <NormalChartToolbox {...i} key={index} />;
+                    })}
+                </ChartToolboxItemContext.Provider>
             </Toolbox>
         </>
     );

frontend/packages/core/src/pages/pr-curve.tsx → frontend/packages/core/src/components/CurvesPage/CurveAside.tsx

@@ -14,21 +14,16 @@
  * limitations under the License.
  */
 
-import ChartPage, {WithChart} from '~/components/ChartPage';
+import type {Run as CurveRun, Tag as CurveTag, CurveType, StepInfo} from '~/resource/curves';
 import React, {FunctionComponent, useCallback, useEffect, useMemo, useState} from 'react';
-import type {Run, StepInfo, Tag} from '~/resource/pr-curve';
 import {rem, transitionProps} from '~/utils/style';
 
 import {AsideSection} from '~/components/Aside';
-import Content from '~/components/Content';
-import Error from '~/components/Error';
 import Field from '~/components/Field';
-import PRCurveChart from '~/components/PRCurvePage/PRCurveChart';
 import RunAside from '~/components/RunAside';
-import StepSlider from '~/components/PRCurvePage/StepSlider';
+import StepSlider from '~/components/CurvesPage/StepSlider';
 import TimeModeSelect from '~/components/TimeModeSelect';
-import {TimeType} from '~/resource/pr-curve';
-import Title from '~/components/Title';
+import {TimeType} from '~/resource/curves';
 import {cycleFetcher} from '~/utils/fetch';
 import queryString from 'query-string';
 import styled from 'styled-components';
@@ -60,12 +55,29 @@ const StepSliderWrapper = styled.div`
     }
 `;
 
-const PRCurve: FunctionComponent = () => {
-    const {t} = useTranslation(['pr-curve', 'common']);
+type CurveAsideProps = {
+    type: CurveType;
+    onChangeLoading: (loading: boolean) => unknown;
+    onChangeSteps: (tags: CurveTag[]) => unknown;
+    onToggleRunning: (running: boolean) => unknown;
+};
+
+const CurveAside: FunctionComponent<CurveAsideProps> = ({type, onChangeLoading, onChangeSteps, onToggleRunning}) => {
+    const {t} = useTranslation('curves');
 
     const [running, setRunning] = useState(true);
 
-    const {runs, tags, runsInTags, selectedRuns, onChangeRuns, loading} = useTagFilter('pr-curve', running);
+    // TODO: remove `as` after ts 4.1
+    const {runs, tags, runsInTags, selectedRuns, onChangeRuns, loading} = useTagFilter(
+        `${type}-curve` as 'pr-curve' | 'roc-curve',
+        running
+    );
+
+    const {data: stepInfo} = useRunningRequest<StepInfo[]>(
+        runsInTags.map(run => `/${type}-curve/steps?${queryString.stringify({run: run.label})}`),
+        !!running,
+        (...urls) => cycleFetcher(urls)
+    );
 
     const [indexes, setIndexes] = useState<Record<string, number>>({});
     const onChangeIndexes = useCallback(
@@ -89,12 +101,7 @@ const PRCurve: FunctionComponent = () => {
         [runsInTags]
     );
 
-    const {data: stepInfo} = useRunningRequest<StepInfo[]>(
-        runsInTags.map(run => `/pr-curve/steps?${queryString.stringify({run: run.label})}`),
-        !!running,
-        (...urls) => cycleFetcher(urls)
-    );
-    const runWithInfo = useMemo<Run[]>(
+    const curveRun = useMemo<CurveRun[]>(
         () =>
             runsInTags.map((run, i) => ({
                 ...run,
@@ -108,70 +115,52 @@ const PRCurve: FunctionComponent = () => {
 
     const [timeType, setTimeType] = useState<TimeType>(TimeType.Step);
 
-    const prCurveTags = useMemo<Tag[]>(
-        () =>
-            tags.map(tag => ({
+    useEffect(() => {
+        onChangeSteps(
+            tags.map<CurveTag>(tag => ({
                 ...tag,
                 runs: tag.runs.map(run => ({
                     ...run,
                     index: 0,
-                    steps: [] as Run['steps'],
-                    wallTimes: [] as Run['wallTimes'],
-                    relatives: [] as Run['relatives'],
-                    ...runWithInfo.find(r => r.label === run.label)
+                    steps: [],
+                    wallTimes: [],
+                    relatives: [],
+                    ...curveRun.find(r => r.label === run.label)
                 }))
-            })),
-        [tags, runWithInfo]
-    );
-
-    const aside = useMemo(
-        () =>
-            runs.length ? (
-                <RunAside
-                    runs={runs}
-                    selectedRuns={selectedRuns}
-                    onChangeRuns={onChangeRuns}
-                    running={running}
-                    onToggleRunning={setRunning}
-                >
-                    <AsideSection>
-                        <Field label={t('pr-curve:time-display-type')}>
-                            <TimeModeSelect value={timeType} onChange={setTimeType} />
-                        </Field>
+            }))
+        );
+    }, [tags, curveRun, onChangeSteps]);
+
+    useEffect(() => {
+        onChangeLoading(loading);
+    }, [loading, onChangeLoading]);
+
+    useEffect(() => {
+        onToggleRunning(running);
+    }, [onToggleRunning, running]);
+
+    return runs.length ? (
+        <RunAside
+            runs={runs}
+            selectedRuns={selectedRuns}
+            onChangeRuns={onChangeRuns}
+            running={running}
+            onToggleRunning={setRunning}
+        >
+            <AsideSection>
+                <Field label={t('curves:time-display-type')}>
+                    <TimeModeSelect value={timeType} onChange={setTimeType} />
+                </Field>
+            </AsideSection>
+            <StepSliderWrapper>
+                {curveRun.map(run => (
+                    <AsideSection key={run.label}>
+                        <StepSlider run={run} type={timeType} onChange={index => onChangeIndexes(run.label, index)} />
                     </AsideSection>
-                    <StepSliderWrapper>
-                        {runWithInfo.map(run => (
-                            <AsideSection key={run.label}>
-                                <StepSlider
-                                    run={run}
-                                    type={timeType}
-                                    onChange={index => onChangeIndexes(run.label, index)}
-                                />
-                            </AsideSection>
-                        ))}
-                    </StepSliderWrapper>
-                </RunAside>
-            ) : null,
-        [t, onChangeRuns, running, runs, selectedRuns, timeType, runWithInfo, onChangeIndexes]
-    );
-
-    const withChart = useCallback<WithChart<Tag>>(
-        ({label, runs, ...args}) => <PRCurveChart runs={runs} tag={label} {...args} running={running} />,
-        [running]
-    );
-
-    return (
-        <>
-            <Title>{t('common:pr-curve')}</Title>
-            <Content aside={aside} loading={loading}>
-                {!loading && !runs.length ? (
-                    <Error />
-                ) : (
-                    <ChartPage items={prCurveTags} withChart={withChart} loading={loading} />
-                )}
-            </Content>
-        </>
-    );
+                ))}
+            </StepSliderWrapper>
+        </RunAside>
+    ) : null;
 };
 
-export default PRCurve;
+export default CurveAside;

frontend/packages/core/src/components/PRCurvePage/PRCurveChart.tsx → frontend/packages/core/src/components/CurvesPage/CurveChart.tsx

@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
+import type {CurveType, PRCurveData, Run} from '~/resource/curves';
 import LineChart, {LineChartRef} from '~/components/LineChart';
-import type {PRCurveData, Run} from '~/resource/pr-curve';
 import React, {FunctionComponent, useCallback, useMemo, useRef, useState} from 'react';
-import {options as chartOptions, nearestPoint} from '~/resource/pr-curve';
+import {options as chartOptions, nearestPoint} from '~/resource/curves';
 import {rem, size} from '~/utils/style';
 
 import ChartToolbox from '~/components/ChartToolbox';
@@ -62,19 +62,20 @@ const Error = styled.div`
 `;
 
 type PRCurveChartProps = {
+    type: CurveType;
     cid: symbol;
     runs: Run[];
     tag: string;
     running?: boolean;
 };
 
-const PRCurveChart: FunctionComponent<PRCurveChartProps> = ({cid, runs, tag, running}) => {
-    const {t} = useTranslation(['pr-curve', 'common']);
+const PRCurveChart: FunctionComponent<PRCurveChartProps> = ({type, cid, runs, tag, running}) => {
+    const {t} = useTranslation(['curves', 'common']);
 
     const echart = useRef<LineChartRef>(null);
 
     const {data: dataset, error, loading} = useRunningRequest<PRCurveData[]>(
-        runs.map(run => `/pr-curve/list?${queryString.stringify({run: run.label, tag})}`),
+        runs.map(run => `/${type}-curve/list?${queryString.stringify({run: run.label, tag})}`),
         !!running,
         (...urls) => cycleFetcher(urls)
     );
@@ -136,25 +137,25 @@ const PRCurveChart: FunctionComponent<PRCurveChartProps> = ({cid, runs, tag, run
             );
             const columns = [
                 {
-                    label: t('pr-curve:threshold')
+                    label: t('curves:threshold')
                 },
                 {
-                    label: t('pr-curve:precision')
+                    label: t('curves:precision')
                 },
                 {
-                    label: t('pr-curve:recall')
+                    label: t('curves:recall')
                 },
                 {
-                    label: t('pr-curve:true-positives')
+                    label: t('curves:true-positives')
                 },
                 {
-                    label: t('pr-curve:false-positives')
+                    label: t('curves:false-positives')
                 },
                 {
-                    label: t('pr-curve:true-negatives')
+                    label: t('curves:true-negatives')
                 },
                 {
-                    label: t('pr-curve:false-negatives')
+                    label: t('curves:false-negatives')
                 }
             ];
             const runData = points.reduce<Run[]>((m, runPoints, index) => {
@@ -208,19 +209,19 @@ const PRCurveChart: FunctionComponent<PRCurveChartProps> = ({cid, runs, tag, run
                     {
                         icon: 'maximize',
                         activeIcon: 'minimize',
-                        tooltip: t('pr-curve:maximize'),
-                        activeTooltip: t('pr-curve:minimize'),
+                        tooltip: t('curves:maximize'),
+                        activeTooltip: t('curves:minimize'),
                         toggle: true,
                         onClick: toggleMaximized
                     },
                     {
                         icon: 'restore-size',
-                        tooltip: t('pr-curve:restore'),
+                        tooltip: t('curves:restore'),
                         onClick: () => echart.current?.restore()
                     },
                     {
                         icon: 'download',
-                        tooltip: t('pr-curve:download-image'),
+                        tooltip: t('curves:download-image'),
                         onClick: () => echart.current?.saveAsImage()
                     }
                 ]}

frontend/packages/core/src/components/PRCurvePage/StepSlider.tsx → frontend/packages/core/src/components/CurvesPage/StepSlider.tsx

@@ -19,8 +19,8 @@ import {ellipsis, size, transitionProps} from '~/utils/style';
 
 import Field from '~/components/Field';
 import RangeSlider from '~/components/RangeSlider';
-import type {Run} from '~/resource/pr-curve';
-import {TimeType} from '~/resource/pr-curve';
+import type {Run} from '~/resource/curves';
+import {TimeType} from '~/resource/curves';
 import {format} from 'd3-format';
 import {formatTime} from '~/utils';
 import styled from 'styled-components';

frontend/packages/core/src/components/HighDimensionalPage/HighDimensionalChart.tsx

@@ -118,10 +118,17 @@ const HighDimensionalChart = React.forwardRef<HighDimensionalChartRef, HighDimen
         useLayoutEffect(() => {
             const c = chartElement.current;
             if (c) {
+                let animationId: number | null = null;
                 const observer = new ResizeObserver(() => {
-                    const rect = c.getBoundingClientRect();
-                    setWidth(rect.width);
-                    setHeight(rect.height);
+                    if (animationId != null) {
+                        cancelAnimationFrame(animationId);
+                        animationId = null;
+                    }
+                    animationId = requestAnimationFrame(() => {
+                        const rect = c.getBoundingClientRect();
+                        setWidth(rect.width);
+                        setHeight(rect.height);
+                    });
                 });
                 observer.observe(c);
                 return () => observer.unobserve(c);

frontend/packages/core/src/components/RunAside.tsx

@@ -83,7 +83,7 @@ const StyledAside = styled(Aside)`
     }
 `;
 
-type RunAsideProps = {
+export type RunAsideProps = {
     runs?: Run[];
     selectedRuns?: Run[];
     onChangeRuns?: (runs: Run[]) => unknown;

frontend/packages/core/src/components/SamplePage/SampleChart.tsx

@@ -23,11 +23,11 @@ import GridLoader from 'react-spinners/GridLoader';
 import type {Run} from '~/types';
 import StepSlider from '~/components/SamplePage/StepSlider';
 import {fetcher} from '~/utils/fetch';
-import fileSaver from 'file-saver';
 import {formatTime} from '~/utils';
 import isEmpty from 'lodash/isEmpty';
 import mime from 'mime-types';
 import queryString from 'query-string';
+import {saveFile} from '~/utils/saveFile';
 import styled from 'styled-components';
 import useRequest from '~/hooks/useRequest';
 import {useRunningRequest} from '~/hooks/useRequest';
@@ -268,13 +268,7 @@ const SampleChart: FunctionComponent<SampleChartProps> = ({
     const download = useCallback(() => {
         if (entityData) {
             const ext = entityData.type ? mime.extension(entityData.type) : null;
-            fileSaver.saveAs(
-                entityData.data,
-                `${run.label}-${tag}-${steps[step]}-${wallTime.toString().replace(/\./, '_')}`.replace(
-                    /[/\\?%*:|"<>]/g,
-                    '_'
-                ) + (ext ? `.${ext}` : '')
-            );
+            saveFile(entityData.data, `${run.label}-${tag}-${steps[step]}-${wallTime}` + (ext ? `.${ext}` : ''));
         }
     }, [entityData, run.label, tag, steps, step, wallTime]);
 

frontend/packages/core/src/components/ScalarPage/ScalarChart.tsx

@@ -39,6 +39,7 @@ import ee from '~/utils/event';
 import {format} from 'd3-format';
 import queryString from 'query-string';
 import {renderToStaticMarkup} from 'react-dom/server';
+import saveFile from '~/utils/saveFile';
 import styled from 'styled-components';
 import {useRunningRequest} from '~/hooks/useRequest';
 import {useTranslation} from 'react-i18next';
@@ -210,6 +211,49 @@ const ScalarChart: FunctionComponent<ScalarChartProps> = ({
         [formatter, ranges, xAxisType, yAxisType]
     );
 
+    const toolbox = useMemo(
+        () => [
+            {
+                icon: 'maximize',
+                activeIcon: 'minimize',
+                tooltip: t('scalar:maximize'),
+                activeTooltip: t('scalar:minimize'),
+                toggle: true,
+                onClick: toggleMaximized
+            },
+            {
+                icon: 'restore-size',
+                tooltip: t('scalar:restore'),
+                onClick: () => echart.current?.restore()
+            },
+            {
+                icon: 'log-axis',
+                tooltip: t('scalar:toggle-log-axis'),
+                toggle: true,
+                onClick: toggleYAxisType
+            },
+            {
+                icon: 'download',
+                menuList: [
+                    {
+                        label: t('scalar:download-image'),
+                        onClick: () => echart.current?.saveAsImage()
+                    },
+                    {
+                        label: t('scalar:download-data'),
+                        onClick: () =>
+                            saveFile(
+                                runs.map(run => `/scalar/data?${queryString.stringify({run: run.label, tag})}`),
+                                runs.map(run => `visualdl-scalar-${run.label}-${tag}.tsv`),
+                                `visualdl-scalar-${tag}.zip`
+                            )
+                    }
+                ]
+            }
+        ],
+        [runs, t, tag, toggleMaximized, toggleYAxisType]
+    );
+
     // display error only on first fetch
     if (!data && error) {
         return <Error>{t('common:error')}</Error>;
@@ -218,34 +262,7 @@ const ScalarChart: FunctionComponent<ScalarChartProps> = ({
     return (
         <Wrapper>
             <StyledLineChart ref={echart} title={tag} options={options} data={data} loading={loading} zoom />
-            <Toolbox
-                items={[
-                    {
-                        icon: 'maximize',
-                        activeIcon: 'minimize',
-                        tooltip: t('scalar:maximize'),
-                        activeTooltip: t('scalar:minimize'),
-                        toggle: true,
-                        onClick: toggleMaximized
-                    },
-                    {
-                        icon: 'restore-size',
-                        tooltip: t('scalar:restore'),
-                        onClick: () => echart.current?.restore()
-                    },
-                    {
-                        icon: 'log-axis',
-                        tooltip: t('scalar:toggle-log-axis'),
-                        toggle: true,
-                        onClick: toggleYAxisType
-                    },
-                    {
-                        icon: 'download',
-                        tooltip: t('scalar:download-image'),
-                        onClick: () => echart.current?.saveAsImage()
-                    }
-                ]}
-            />
+            <Toolbox items={toolbox} />
         </Wrapper>
     );
 };

frontend/packages/core/src/hooks/useECharts.ts

@@ -21,7 +21,7 @@ import {position, primaryColor, size} from '~/utils/style';
 
 import type {ECharts} from 'echarts';
 import {dataURL2Blob} from '~/utils/image';
-import fileSaver from 'file-saver';
+import {saveFile} from '~/utils/saveFile';
 import styled from 'styled-components';
 import {themes} from '~/utils/theme';
 import useTheme from '~/hooks/useTheme';
@@ -118,8 +118,15 @@ const useECharts = <T extends HTMLElement, W extends HTMLElement = HTMLDivElemen
         if (options.autoFit) {
             const w = wrapper.current;
             if (w) {
+                let animationId: number | null = null;
                 const observer = new ResizeObserver(() => {
-                    echart?.resize();
+                    if (animationId != null) {
+                        cancelAnimationFrame(animationId);
+                        animationId = null;
+                    }
+                    animationId = requestAnimationFrame(() => {
+                        echart?.resize();
+                    });
                 });
                 observer.observe(w);
                 return () => observer.unobserve(w);
@@ -131,7 +138,7 @@ const useECharts = <T extends HTMLElement, W extends HTMLElement = HTMLDivElemen
         (filename?: string) => {
             if (echart) {
                 const blob = dataURL2Blob(echart.getDataURL({type: 'png', pixelRatio: 2, backgroundColor: '#FFF'}));
-                fileSaver.saveAs(blob, `${filename?.replace(/[/\\?%*:|"<>]/g, '_') || 'chart'}.png`);
+                saveFile(blob, `${filename || 'chart'}.png`);
             }
         },
         [echart]

frontend/packages/core/src/hooks/useNavItems.ts

@@ -28,7 +28,8 @@ export const navMap = {
     audio: Pages.Audio,
     graph: Pages.Graph,
     embeddings: Pages.HighDimensional,
-    pr_curve: Pages.PRCurve
+    pr_curve: Pages.PRCurve,
+    roc_curve: Pages.ROCCurve
 } as const;
 
 const useNavItems = () => {

frontend/packages/core/src/pages/curves/pr.tsx

+/**
+ * Copyright 2020 Baidu Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import ChartPage, {WithChart} from '~/components/ChartPage';
+import React, {FunctionComponent, useCallback, useState} from 'react';
+
+import Content from '~/components/Content';
+import CurveAside from '~/components/CurvesPage/CurveAside';
+import CurveChart from '~/components/CurvesPage/CurveChart';
+import Error from '~/components/Error';
+import type {Tag} from '~/resource/curves';
+import Title from '~/components/Title';
+import {useTranslation} from 'react-i18next';
+
+const PRCurve: FunctionComponent = () => {
+    const {t} = useTranslation('common');
+
+    const [running, setRunning] = useState(true);
+    const [loading, setLoading] = useState(false);
+
+    const [tags, setTags] = useState<Tag[]>([]);
+
+    const withChart = useCallback<WithChart<Tag>>(
+        ({label, runs, ...args}) => <CurveChart type="pr" runs={runs} tag={label} {...args} running={running} />,
+        [running]
+    );
+
+    return (
+        <>
+            <Title>{t('common:pr-curve')}</Title>
+            <Content
+                aside={
+                    <CurveAside
+                        type="pr"
+                        onChangeLoading={setLoading}
+                        onChangeSteps={setTags}
+                        onToggleRunning={setRunning}
+                    />
+                }
+                loading={loading}
+            >
+                {!loading && !tags.length ? (
+                    <Error />
+                ) : (
+                    <ChartPage items={tags} withChart={withChart} loading={loading} />
+                )}
+            </Content>
+        </>
+    );
+};
+
+export default PRCurve;

frontend/packages/core/src/pages/curves/roc.tsx

+/**
+ * Copyright 2020 Baidu Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import ChartPage, {WithChart} from '~/components/ChartPage';
+import React, {FunctionComponent, useCallback, useState} from 'react';
+
+import Content from '~/components/Content';
+import CurveAside from '~/components/CurvesPage/CurveAside';
+import CurveChart from '~/components/CurvesPage/CurveChart';
+import Error from '~/components/Error';
+import type {Tag} from '~/resource/curves';
+import Title from '~/components/Title';
+import {useTranslation} from 'react-i18next';
+
+const ROCCurve: FunctionComponent = () => {
+    const {t} = useTranslation('common');
+
+    const [running, setRunning] = useState(true);
+    const [loading, setLoading] = useState(false);
+
+    const [tags, setTags] = useState<Tag[]>([]);
+
+    const withChart = useCallback<WithChart<Tag>>(
+        ({label, runs, ...args}) => <CurveChart type="roc" runs={runs} tag={label} {...args} running={running} />,
+        [running]
+    );
+
+    return (
+        <>
+            <Title>{t('common:roc-curve')}</Title>
+            <Content
+                aside={
+                    <CurveAside
+                        type="roc"
+                        onChangeLoading={setLoading}
+                        onChangeSteps={setTags}
+                        onToggleRunning={setRunning}
+                    />
+                }
+                loading={loading}
+            >
+                {!loading && !tags.length ? (
+                    <Error />
+                ) : (
+                    <ChartPage items={tags} withChart={withChart} loading={loading} />
+                )}
+            </Content>
+        </>
+    );
+};
+
+export default ROCCurve;

frontend/packages/core/src/pages/sample/audio.tsx

@@ -32,7 +32,7 @@ const chartSize = {
     height: rem(244)
 };
 
-const Audio: FunctionComponent = () => {
+const AudioSample: FunctionComponent = () => {
     const {t} = useTranslation(['sample', 'common']);
 
     const audioContext = useRef<AudioContext>();
@@ -90,4 +90,4 @@ const Audio: FunctionComponent = () => {
     );
 };
 
-export default Audio;
+export default AudioSample;

frontend/packages/core/src/pages/sample/image.tsx

@@ -36,7 +36,7 @@ const chartSize = {
     height: rem(406)
 };
 
-const Image: FunctionComponent = () => {
+const ImageSample: FunctionComponent = () => {
     const {t} = useTranslation(['sample', 'common']);
 
     const [running, setRunning] = useState(true);
@@ -112,4 +112,4 @@ const Image: FunctionComponent = () => {
     );
 };
 
-export default Image;
+export default ImageSample;

frontend/packages/core/src/resource/pr-curve/index.ts → frontend/packages/core/src/resource/curves/index.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-export type {PRCurveData, Run, StepInfo, Tag} from './types';
+export type {CurveType, PRCurveData, Run, StepInfo, Tag} from './types';
 export {TimeType} from './types';
 export * from './chart';
 export * from './data';

frontend/packages/core/src/resource/pr-curve/types.ts → frontend/packages/core/src/resource/curves/types.ts

@@ -18,6 +18,8 @@ import {Run as BaseRun, Tag as BaseTag, TimeMode} from '~/types';
 
 export {TimeMode as TimeType};
 
+export type CurveType = 'pr' | 'roc';
+
 type Step = number;
 type WallTime = number;
 type Relative = number;

frontend/packages/core/src/routes/index.ts

@@ -23,7 +23,8 @@ export enum Pages {
     Audio = 'audio',
     Graph = 'graph',
     HighDimensional = 'high-dimensional',
-    PRCurve = 'pr-curve'
+    PRCurve = 'pr-curve',
+    ROCCurve = 'roc-curve'
 }
 
 export interface Route {
@@ -81,7 +82,12 @@ const routes: Route[] = [
     {
         id: Pages.PRCurve,
         path: '/pr-curve',
-        component: React.lazy(() => import('~/pages/pr-curve'))
+        component: React.lazy(() => import('~/pages/curves/pr'))
+    },
+    {
+        id: Pages.ROCCurve,
+        path: '/roc-curve',
+        component: React.lazy(() => import('~/pages/curves/roc'))
     }
 ];
 

frontend/packages/core/src/store/runs/reducers.ts

@@ -23,7 +23,8 @@ const initState: RunsState = {
     histogram: [],
     image: [],
     audio: [],
-    'pr-curve': []
+    'pr-curve': [],
+    'roc-curve': []
 };
 
 function runsReducer(state = initState, action: RunsActionTypes): RunsState {

frontend/packages/core/src/store/runs/types.ts

@@ -26,6 +26,7 @@ export interface RunsState {
     image: Runs;
     audio: Runs;
     'pr-curve': Runs;
+    'roc-curve': Runs;
 }
 
 export type Page = keyof RunsState;

frontend/packages/core/src/utils/saveFile.ts

+/**
+ * Copyright 2020 Baidu Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import FileSaver from 'file-saver';
+import JSZip from 'jszip';
+import {fetcher} from '~/utils/fetch';
+import isString from 'lodash/isString';
+import {toast} from 'react-toastify';
+
+async function getFile(url: string): Promise<string | Blob> {
+    const result = await fetcher(url);
+    if (result.data instanceof Blob) {
+        return result.data;
+    } else if (!isString(result)) {
+        return JSON.stringify(result);
+    }
+    return result;
+}
+
+function normalizeFilename(name: string) {
+    return name.replace(/[/\\?%*:|"<>]/g, '_');
+}
+
+export function saveFile(file: string | Blob, filename: string) {
+    let blob: Blob;
+    if (file instanceof Blob) {
+        blob = file;
+    } else {
+        blob = new Blob([file], {type: 'text/plain;charset=utf-8'});
+    }
+    FileSaver.saveAs(blob, normalizeFilename(filename));
+}
+
+export default async function (url: string | string[], filename: string | string[], zipFilename = 'download.zip') {
+    if (!url) {
+        return;
+    }
+    let urls: string[] = url as string[];
+    let filenames: string[] = filename as string[];
+    if (isString(url)) {
+        urls = [url];
+    }
+    if (isString(filename)) {
+        filenames = [filename];
+    }
+    try {
+        const data = await Promise.all(urls.map(getFile));
+        if (data.length === 1) {
+            saveFile(data[0], filenames[0]);
+        } else {
+            const zip = new JSZip();
+            let basename = '';
+            let extname = '';
+            if (filenames.length === 1) {
+                const filenameArr = filenames[0].split('.');
+                if (filenameArr.length > 1) {
+                    extname = filenameArr.pop() as string;
+                }
+                basename = filenameArr.join('.');
+            }
+            data.forEach((file, index) => {
+                zip.file(normalizeFilename(basename ? `${basename} - ${index}.${extname}` : filenames[index]), file);
+            });
+            const zipFile = await zip.generateAsync({type: 'blob'});
+            saveFile(zipFile, zipFilename);
+        }
+    } catch (e) {
+        toast(e.message, {
+            position: toast.POSITION.TOP_CENTER,
+            type: toast.TYPE.ERROR
+        });
+    }
+}

frontend/packages/core/src/utils/style.ts

@@ -149,7 +149,7 @@ export const GlobalStyle = createGlobalStyle`
 
     html,
     body {
-        height: 100%;
+        min-height: 100%;
         background-color: var(--body-background-color);
         color: var(--text-color);
         ${transitionProps(['background-color', 'color'])}

frontend/packages/demo/builder/index.ts

@@ -27,7 +27,7 @@ import {spawn} from 'child_process';
 
 const host = '127.0.0.1';
 const publicPath = '/visualdl';
-const pages = ['common', 'scalar', 'histogram', 'image', 'audio', 'graph', 'pr-curve', 'high-dimensional'];
+const pages = ['common', 'scalar', 'histogram', 'image', 'audio', 'graph', 'pr-curve', 'roc-curve', 'high-dimensional'];
 const dataDir = path.resolve(__dirname, '../data');
 
 async function start() {

frontend/packages/demo/builder/roc-curve.ts

+/**
+ * Copyright 2020 Baidu Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type {Data, Worker} from './types';
+
+const worker: Worker = async io => {
+    const components = await io.getData<string[]>('/components');
+    if (!components.includes('roc_curve')) {
+        return;
+    }
+
+    const {runs, tags} = await io.save<Data>('/roc-curve/tags');
+    for (const [index, run] of runs.entries()) {
+        await io.save('/roc-curve/steps', {run});
+        for (const tag of tags[index]) {
+            await io.save('/roc-curve/list', {run, tag});
+        }
+    }
+};
+
+export default worker;

frontend/packages/mock/data/components.ts

@@ -14,4 +14,4 @@
  * limitations under the License.
  */
 
-export default ['embeddings', 'scalar', 'image', 'audio', 'graph', 'histogram', 'pr_curve'];
+export default ['embeddings', 'scalar', 'image', 'audio', 'graph', 'histogram', 'pr_curve', 'roc_curve'];

frontend/packages/mock/data/roc-curve/list.ts

+/**
+ * Copyright 2020 Baidu Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Request} from 'express';
+
+export default (request: Request) => {
+    if (request.query.run === 'train') {
+        return [
+            [
+                1593069993786.464,
+                0,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3348623812198639,
+                    0.35121950507164,
+                    0.3604061007499695,
+                    0.3729729652404785,
+                    0.3720930218696594,
+                    0.3899371027946472,
+                    0.3986486494541168,
+                    0.40740740299224854,
+                    0.4065040647983551,
+                    0.43396225571632385,
+                    0.47252747416496277,
+                    0.47058823704719543,
+                    0.5066666603088379,
+                    0.48571428656578064,
+                    0.5,
+                    0.5319148898124695,
+                    0.5121951103210449,
+                    0.5555555820465088,
+                    0.692307710647583,
+                    0.739130437374115,
+                    0.761904776096344,
+                    0.8125,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9866666793823242,
+                    0.9733333587646484,
+                    0.9599999785423279,
+                    0.9466666579246521,
+                    0.9200000166893005,
+                    0.8533333539962769,
+                    0.8266666531562805,
+                    0.7866666913032532,
+                    0.7333333492279053,
+                    0.6666666865348816,
+                    0.6133333444595337,
+                    0.5733333230018616,
+                    0.5333333611488342,
+                    0.5066666603088379,
+                    0.4533333480358124,
+                    0.41333332657814026,
+                    0.3333333432674408,
+                    0.2800000011920929,
+                    0.2666666805744171,
+                    0.23999999463558197,
+                    0.2266666740179062,
+                    0.2133333384990692,
+                    0.1733333319425583,
+                    0.1599999964237213,
+                    0.13333334028720856,
+                    0.1066666692495346,
+                    0.09333333373069763,
+                    0.07999999821186066,
+                    0.07999999821186066,
+                    0.07999999821186066,
+                    0.06666667014360428,
+                    0.0533333346247673,
+                    0.02666666731238365,
+                    0.013333333656191826
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    74,
+                    73,
+                    72,
+                    71,
+                    69,
+                    64,
+                    62,
+                    59,
+                    55,
+                    50,
+                    46,
+                    43,
+                    40,
+                    38,
+                    34,
+                    31,
+                    25,
+                    21,
+                    20,
+                    18,
+                    17,
+                    16,
+                    13,
+                    12,
+                    10,
+                    8,
+                    7,
+                    6,
+                    6,
+                    6,
+                    5,
+                    4,
+                    2,
+                    1
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    148,
+                    145,
+                    133,
+                    126,
+                    116,
+                    108,
+                    97,
+                    89,
+                    80,
+                    73,
+                    60,
+                    48,
+                    45,
+                    37,
+                    36,
+                    31,
+                    22,
+                    20,
+                    16,
+                    8,
+                    6,
+                    5,
+                    3,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    2,
+                    5,
+                    17,
+                    24,
+                    34,
+                    42,
+                    53,
+                    61,
+                    70,
+                    77,
+                    90,
+                    102,
+                    105,
+                    113,
+                    114,
+                    119,
+                    128,
+                    130,
+                    134,
+                    142,
+                    144,
+                    145,
+                    147,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    2,
+                    3,
+                    4,
+                    6,
+                    11,
+                    13,
+                    16,
+                    20,
+                    25,
+                    29,
+                    32,
+                    35,
+                    37,
+                    41,
+                    44,
+                    50,
+                    54,
+                    55,
+                    57,
+                    58,
+                    59,
+                    62,
+                    63,
+                    65,
+                    67,
+                    68,
+                    69,
+                    69,
+                    69,
+                    70,
+                    71,
+                    73,
+                    74
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82
+                ]
+            ],
+            [
+                1593069993787.353,
+                1,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3348214328289032,
+                    0.3333333432674408,
+                    0.3509615361690521,
+                    0.36000001430511475,
+                    0.3776595890522003,
+                    0.395480215549469,
+                    0.386904776096344,
+                    0.40816327929496765,
+                    0.42335766553878784,
+                    0.4193548262119293,
+                    0.4285714328289032,
+                    0.43119266629219055,
+                    0.4536082446575165,
+                    0.46666666865348816,
+                    0.4749999940395355,
+                    0.4864864945411682,
+                    0.46875,
+                    0.4912280738353729,
+                    0.52173912525177,
+                    0.4871794879436493,
+                    0.5,
+                    0.4482758641242981,
+                    0.4399999976158142,
+                    0.40909090638160706,
+                    0.4117647111415863,
+                    0.4375,
+                    0.4166666567325592,
+                    0.5555555820465088,
+                    0.5,
+                    0.5,
+                    0.25,
+                    0.5,
+                    0.5,
+                    0.5,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9733333587646484,
+                    0.9733333587646484,
+                    0.9599999785423279,
+                    0.9466666579246521,
+                    0.9333333373069763,
+                    0.8666666746139526,
+                    0.800000011920929,
+                    0.7733333110809326,
+                    0.6933333277702332,
+                    0.6399999856948853,
+                    0.6266666650772095,
+                    0.5866666436195374,
+                    0.5600000023841858,
+                    0.5066666603088379,
+                    0.47999998927116394,
+                    0.4000000059604645,
+                    0.3733333349227905,
+                    0.3199999928474426,
+                    0.25333333015441895,
+                    0.23999999463558197,
+                    0.1733333319425583,
+                    0.14666666090488434,
+                    0.11999999731779099,
+                    0.09333333373069763,
+                    0.09333333373069763,
+                    0.06666667014360428,
+                    0.06666667014360428,
+                    0.0533333346247673,
+                    0.03999999910593033,
+                    0.013333333656191826,
+                    0.013333333656191826,
+                    0.013333333656191826,
+                    0.013333333656191826,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    73,
+                    73,
+                    72,
+                    71,
+                    70,
+                    65,
+                    60,
+                    58,
+                    52,
+                    48,
+                    47,
+                    44,
+                    42,
+                    38,
+                    36,
+                    30,
+                    28,
+                    24,
+                    19,
+                    18,
+                    13,
+                    11,
+                    9,
+                    7,
+                    7,
+                    5,
+                    5,
+                    4,
+                    3,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    149,
+                    146,
+                    135,
+                    128,
+                    117,
+                    107,
+                    103,
+                    87,
+                    79,
+                    72,
+                    64,
+                    62,
+                    53,
+                    48,
+                    42,
+                    38,
+                    34,
+                    29,
+                    22,
+                    20,
+                    18,
+                    16,
+                    14,
+                    13,
+                    10,
+                    9,
+                    7,
+                    4,
+                    4,
+                    3,
+                    3,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    4,
+                    15,
+                    22,
+                    33,
+                    43,
+                    47,
+                    63,
+                    71,
+                    78,
+                    86,
+                    88,
+                    97,
+                    102,
+                    108,
+                    112,
+                    116,
+                    121,
+                    128,
+                    130,
+                    132,
+                    134,
+                    136,
+                    137,
+                    140,
+                    141,
+                    143,
+                    146,
+                    146,
+                    147,
+                    147,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    2,
+                    2,
+                    3,
+                    4,
+                    5,
+                    10,
+                    15,
+                    17,
+                    23,
+                    27,
+                    28,
+                    31,
+                    33,
+                    37,
+                    39,
+                    45,
+                    47,
+                    51,
+                    56,
+                    57,
+                    62,
+                    64,
+                    66,
+                    68,
+                    68,
+                    70,
+                    70,
+                    71,
+                    72,
+                    74,
+                    74,
+                    74,
+                    74,
+                    75,
+                    75,
+                    75
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82,
+                    0.84,
+                    0.86
+                ]
+            ],
+            [
+                1593069993788.1448,
+                2,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3348214328289032,
+                    0.33181819319725037,
+                    0.33796295523643494,
+                    0.34272301197052,
+                    0.3497537076473236,
+                    0.3526315689086914,
+                    0.36781609058380127,
+                    0.3836477994918823,
+                    0.3959731459617615,
+                    0.3893129825592041,
+                    0.3613445460796356,
+                    0.37383177876472473,
+                    0.3814432919025421,
+                    0.42352941632270813,
+                    0.43661972880363464,
+                    0.4029850661754608,
+                    0.4166666567325592,
+                    0.4038461446762085,
+                    0.41304346919059753,
+                    0.4749999940395355,
+                    0.4571428596973419,
+                    0.4137931168079376,
+                    0.3913043439388275,
+                    0.44999998807907104,
+                    0.47058823704719543,
+                    0.5,
+                    0.5384615659713745,
+                    0.7777777910232544,
+                    0.8571428656578064,
+                    0.800000011920929,
+                    0.75,
+                    0.75,
+                    0.6666666865348816,
+                    0.5,
+                    0,
+                    0
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9733333587646484,
+                    0.9733333587646484,
+                    0.9733333587646484,
+                    0.9466666579246521,
+                    0.8933333158493042,
+                    0.8533333539962769,
+                    0.8133333325386047,
+                    0.7866666913032532,
+                    0.6800000071525574,
+                    0.5733333230018616,
+                    0.5333333611488342,
+                    0.4933333396911621,
+                    0.47999998927116394,
+                    0.41333332657814026,
+                    0.36000001430511475,
+                    0.3333333432674408,
+                    0.2800000011920929,
+                    0.25333333015441895,
+                    0.25333333015441895,
+                    0.2133333384990692,
+                    0.1599999964237213,
+                    0.11999999731779099,
+                    0.11999999731779099,
+                    0.1066666692495346,
+                    0.09333333373069763,
+                    0.09333333373069763,
+                    0.09333333373069763,
+                    0.07999999821186066,
+                    0.0533333346247673,
+                    0.03999999910593033,
+                    0.03999999910593033,
+                    0.02666666731238365,
+                    0.013333333656191826,
+                    0,
+                    0
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    73,
+                    73,
+                    73,
+                    71,
+                    67,
+                    64,
+                    61,
+                    59,
+                    51,
+                    43,
+                    40,
+                    37,
+                    36,
+                    31,
+                    27,
+                    25,
+                    21,
+                    19,
+                    19,
+                    16,
+                    12,
+                    9,
+                    9,
+                    8,
+                    7,
+                    7,
+                    7,
+                    6,
+                    4,
+                    3,
+                    3,
+                    2,
+                    1,
+                    0,
+                    0
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    149,
+                    147,
+                    143,
+                    140,
+                    132,
+                    123,
+                    110,
+                    98,
+                    90,
+                    80,
+                    76,
+                    67,
+                    60,
+                    49,
+                    40,
+                    40,
+                    35,
+                    31,
+                    27,
+                    21,
+                    19,
+                    17,
+                    14,
+                    11,
+                    9,
+                    7,
+                    6,
+                    2,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    3,
+                    7,
+                    10,
+                    18,
+                    27,
+                    40,
+                    52,
+                    60,
+                    70,
+                    74,
+                    83,
+                    90,
+                    101,
+                    110,
+                    110,
+                    115,
+                    119,
+                    123,
+                    129,
+                    131,
+                    133,
+                    136,
+                    139,
+                    141,
+                    143,
+                    144,
+                    148,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    2,
+                    2,
+                    2,
+                    4,
+                    8,
+                    11,
+                    14,
+                    16,
+                    24,
+                    32,
+                    35,
+                    38,
+                    39,
+                    44,
+                    48,
+                    50,
+                    54,
+                    56,
+                    56,
+                    59,
+                    63,
+                    66,
+                    66,
+                    67,
+                    68,
+                    68,
+                    68,
+                    69,
+                    71,
+                    72,
+                    72,
+                    73,
+                    74,
+                    75,
+                    75
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82
+                ]
+            ],
+            [
+                1593069993788.836,
+                3,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3303571343421936,
+                    0.32407405972480774,
+                    0.32380953431129456,
+                    0.3350253701210022,
+                    0.34224599599838257,
+                    0.36627906560897827,
+                    0.3459119498729706,
+                    0.36986300349235535,
+                    0.375,
+                    0.3739837408065796,
+                    0.38532111048698425,
+                    0.4040403962135315,
+                    0.42696627974510193,
+                    0.45121949911117554,
+                    0.47826087474823,
+                    0.49152541160583496,
+                    0.49056604504585266,
+                    0.4897959232330322,
+                    0.4651162922382355,
+                    0.4749999940395355,
+                    0.5,
+                    0.48275861144065857,
+                    0.5,
+                    0.4761904776096344,
+                    0.5,
+                    0.47058823704719543,
+                    0.5714285969734192,
+                    0.5454545617103577,
+                    0.5555555820465088,
+                    0.5,
+                    0.5714285969734192,
+                    0.5714285969734192,
+                    0.25,
+                    0.3333333432674408,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9866666793823242,
+                    0.9333333373069763,
+                    0.9066666960716248,
+                    0.8799999952316284,
+                    0.8533333539962769,
+                    0.8399999737739563,
+                    0.7333333492279053,
+                    0.7200000286102295,
+                    0.6800000071525574,
+                    0.6133333444595337,
+                    0.5600000023841858,
+                    0.5333333611488342,
+                    0.5066666603088379,
+                    0.4933333396911621,
+                    0.4399999976158142,
+                    0.3866666555404663,
+                    0.3466666638851166,
+                    0.3199999928474426,
+                    0.2666666805744171,
+                    0.25333333015441895,
+                    0.23999999463558197,
+                    0.18666666746139526,
+                    0.14666666090488434,
+                    0.13333334028720856,
+                    0.13333334028720856,
+                    0.1066666692495346,
+                    0.1066666692495346,
+                    0.07999999821186066,
+                    0.06666667014360428,
+                    0.0533333346247673,
+                    0.0533333346247673,
+                    0.0533333346247673,
+                    0.013333333656191826,
+                    0.013333333656191826,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    74,
+                    70,
+                    68,
+                    66,
+                    64,
+                    63,
+                    55,
+                    54,
+                    51,
+                    46,
+                    42,
+                    40,
+                    38,
+                    37,
+                    33,
+                    29,
+                    26,
+                    24,
+                    20,
+                    19,
+                    18,
+                    14,
+                    11,
+                    10,
+                    10,
+                    8,
+                    8,
+                    6,
+                    5,
+                    4,
+                    4,
+                    4,
+                    1,
+                    1,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    146,
+                    142,
+                    131,
+                    123,
+                    109,
+                    104,
+                    92,
+                    85,
+                    77,
+                    67,
+                    59,
+                    51,
+                    45,
+                    36,
+                    30,
+                    27,
+                    25,
+                    23,
+                    21,
+                    18,
+                    15,
+                    11,
+                    11,
+                    10,
+                    9,
+                    6,
+                    5,
+                    4,
+                    4,
+                    3,
+                    3,
+                    3,
+                    2,
+                    2,
+                    1,
+                    1,
+                    1,
+                    1
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    4,
+                    8,
+                    19,
+                    27,
+                    41,
+                    46,
+                    58,
+                    65,
+                    73,
+                    83,
+                    91,
+                    99,
+                    105,
+                    114,
+                    120,
+                    123,
+                    125,
+                    127,
+                    129,
+                    132,
+                    135,
+                    139,
+                    139,
+                    140,
+                    141,
+                    144,
+                    145,
+                    146,
+                    146,
+                    147,
+                    147,
+                    147,
+                    148,
+                    148,
+                    149,
+                    149,
+                    149,
+                    149
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    5,
+                    7,
+                    9,
+                    11,
+                    12,
+                    20,
+                    21,
+                    24,
+                    29,
+                    33,
+                    35,
+                    37,
+                    38,
+                    42,
+                    46,
+                    49,
+                    51,
+                    55,
+                    56,
+                    57,
+                    61,
+                    64,
+                    65,
+                    65,
+                    67,
+                    67,
+                    69,
+                    70,
+                    71,
+                    71,
+                    71,
+                    74,
+                    74,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82,
+                    0.84,
+                    0.86,
+                    0.88,
+                    0.9
+                ]
+            ],
+            [
+                1593069993789.4,
+                4,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3348214328289032,
+                    0.3363228738307953,
+                    0.3317972421646118,
+                    0.33816424012184143,
+                    0.3484848439693451,
+                    0.3492063581943512,
+                    0.3636363744735718,
+                    0.363095223903656,
+                    0.3717948794364929,
+                    0.37142857909202576,
+                    0.3840000033378601,
+                    0.3839285671710968,
+                    0.3979591727256775,
+                    0.4166666567325592,
+                    0.4722222089767456,
+                    0.460317462682724,
+                    0.4912280738353729,
+                    0.4901960790157318,
+                    0.5333333611488342,
+                    0.5641025900840759,
+                    0.5882353186607361,
+                    0.6666666865348816,
+                    0.625,
+                    0.5555555820465088,
+                    0.5625,
+                    0.6363636255264282,
+                    0.6666666865348816,
+                    0.6666666865348816,
+                    0.5,
+                    0.5,
+                    0.5,
+                    0.6666666865348816,
+                    1,
+                    1
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9599999785423279,
+                    0.9333333373069763,
+                    0.9200000166893005,
+                    0.8799999952316284,
+                    0.8533333539962769,
+                    0.8133333325386047,
+                    0.7733333110809326,
+                    0.6933333277702332,
+                    0.6399999856948853,
+                    0.5733333230018616,
+                    0.5199999809265137,
+                    0.46666666865348816,
+                    0.4533333480358124,
+                    0.3866666555404663,
+                    0.3733333349227905,
+                    0.3333333432674408,
+                    0.3199999928474426,
+                    0.2933333218097687,
+                    0.2666666805744171,
+                    0.23999999463558197,
+                    0.20000000298023224,
+                    0.13333334028720856,
+                    0.11999999731779099,
+                    0.09333333373069763,
+                    0.07999999821186066,
+                    0.0533333346247673,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.013333333656191826
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    72,
+                    70,
+                    69,
+                    66,
+                    64,
+                    61,
+                    58,
+                    52,
+                    48,
+                    43,
+                    39,
+                    35,
+                    34,
+                    29,
+                    28,
+                    25,
+                    24,
+                    22,
+                    20,
+                    18,
+                    15,
+                    10,
+                    9,
+                    7,
+                    6,
+                    4,
+                    2,
+                    2,
+                    2,
+                    2,
+                    2,
+                    1
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    149,
+                    148,
+                    145,
+                    137,
+                    129,
+                    123,
+                    112,
+                    107,
+                    98,
+                    88,
+                    77,
+                    69,
+                    59,
+                    49,
+                    38,
+                    34,
+                    29,
+                    26,
+                    21,
+                    17,
+                    14,
+                    9,
+                    9,
+                    8,
+                    7,
+                    4,
+                    3,
+                    2,
+                    2,
+                    2,
+                    2,
+                    1,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    2,
+                    5,
+                    13,
+                    21,
+                    27,
+                    38,
+                    43,
+                    52,
+                    62,
+                    73,
+                    81,
+                    91,
+                    101,
+                    112,
+                    116,
+                    121,
+                    124,
+                    129,
+                    133,
+                    136,
+                    141,
+                    141,
+                    142,
+                    143,
+                    146,
+                    147,
+                    148,
+                    148,
+                    148,
+                    148,
+                    149,
+                    150,
+                    150
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    3,
+                    5,
+                    6,
+                    9,
+                    11,
+                    14,
+                    17,
+                    23,
+                    27,
+                    32,
+                    36,
+                    40,
+                    41,
+                    46,
+                    47,
+                    50,
+                    51,
+                    53,
+                    55,
+                    57,
+                    60,
+                    65,
+                    66,
+                    68,
+                    69,
+                    71,
+                    73,
+                    73,
+                    73,
+                    73,
+                    73,
+                    74
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78
+                ]
+            ],
+            [
+                1593069993790.076,
+                5,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3348214328289032,
+                    0.3378378450870514,
+                    0.3410138189792633,
+                    0.3507108986377716,
+                    0.3631840944290161,
+                    0.3617021143436432,
+                    0.3604651093482971,
+                    0.36250001192092896,
+                    0.3758389353752136,
+                    0.3834586441516876,
+                    0.4000000059604645,
+                    0.3980582654476166,
+                    0.44186046719551086,
+                    0.4430379867553711,
+                    0.48571428656578064,
+                    0.515625,
+                    0.5892857313156128,
+                    0.5490196347236633,
+                    0.5714285969734192,
+                    0.5641025900840759,
+                    0.5588235259056091,
+                    0.5862069129943848,
+                    0.5652173757553101,
+                    0.6000000238418579,
+                    0.6666666865348816,
+                    0.6470588445663452,
+                    0.6875,
+                    0.6666666865348816,
+                    0.6666666865348816,
+                    0.6666666865348816,
+                    0.75,
+                    0.7142857313156128,
+                    0.5,
+                    0.5,
+                    0.5,
+                    1
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9866666793823242,
+                    0.9866666793823242,
+                    0.9733333587646484,
+                    0.9066666960716248,
+                    0.8266666531562805,
+                    0.7733333110809326,
+                    0.746666669845581,
+                    0.6800000071525574,
+                    0.6133333444595337,
+                    0.54666668176651,
+                    0.5066666603088379,
+                    0.46666666865348816,
+                    0.4533333480358124,
+                    0.4399999976158142,
+                    0.4399999976158142,
+                    0.3733333349227905,
+                    0.3199999928474426,
+                    0.2933333218097687,
+                    0.25333333015441895,
+                    0.2266666740179062,
+                    0.1733333319425583,
+                    0.1599999964237213,
+                    0.1599999964237213,
+                    0.14666666090488434,
+                    0.14666666090488434,
+                    0.13333334028720856,
+                    0.1066666692495346,
+                    0.1066666692495346,
+                    0.07999999821186066,
+                    0.06666667014360428,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.013333333656191826,
+                    0.013333333656191826
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    74,
+                    74,
+                    73,
+                    68,
+                    62,
+                    58,
+                    56,
+                    51,
+                    46,
+                    41,
+                    38,
+                    35,
+                    34,
+                    33,
+                    33,
+                    28,
+                    24,
+                    22,
+                    19,
+                    17,
+                    13,
+                    12,
+                    12,
+                    11,
+                    11,
+                    10,
+                    8,
+                    8,
+                    6,
+                    5,
+                    2,
+                    2,
+                    1,
+                    1
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    149,
+                    147,
+                    143,
+                    137,
+                    128,
+                    120,
+                    110,
+                    102,
+                    93,
+                    82,
+                    69,
+                    62,
+                    48,
+                    44,
+                    36,
+                    31,
+                    23,
+                    23,
+                    18,
+                    17,
+                    15,
+                    12,
+                    10,
+                    8,
+                    6,
+                    6,
+                    5,
+                    5,
+                    4,
+                    4,
+                    2,
+                    2,
+                    2,
+                    2,
+                    1,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    3,
+                    7,
+                    13,
+                    22,
+                    30,
+                    40,
+                    48,
+                    57,
+                    68,
+                    81,
+                    88,
+                    102,
+                    106,
+                    114,
+                    119,
+                    127,
+                    127,
+                    132,
+                    133,
+                    135,
+                    138,
+                    140,
+                    142,
+                    144,
+                    144,
+                    145,
+                    145,
+                    146,
+                    146,
+                    148,
+                    148,
+                    148,
+                    148,
+                    149,
+                    150
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    1,
+                    2,
+                    7,
+                    13,
+                    17,
+                    19,
+                    24,
+                    29,
+                    34,
+                    37,
+                    40,
+                    41,
+                    42,
+                    42,
+                    47,
+                    51,
+                    53,
+                    56,
+                    58,
+                    62,
+                    63,
+                    63,
+                    64,
+                    64,
+                    65,
+                    67,
+                    67,
+                    69,
+                    70,
+                    73,
+                    73,
+                    74,
+                    74
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82
+                ]
+            ],
+            [
+                1593069993790.763,
+                6,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.33636364340782166,
+                    0.3396226465702057,
+                    0.34299516677856445,
+                    0.34825870394706726,
+                    0.35638296604156494,
+                    0.3641618490219116,
+                    0.3836477994918823,
+                    0.4041095972061157,
+                    0.4148148000240326,
+                    0.4406779706478119,
+                    0.4615384638309479,
+                    0.4285714328289032,
+                    0.44871795177459717,
+                    0.47826087474823,
+                    0.4769230782985687,
+                    0.4655172526836395,
+                    0.4901960790157318,
+                    0.54347825050354,
+                    0.523809552192688,
+                    0.5249999761581421,
+                    0.6000000238418579,
+                    0.5555555820465088,
+                    0.5416666865348816,
+                    0.5263158082962036,
+                    0.47058823704719543,
+                    0.3571428656578064,
+                    0.4545454680919647,
+                    0.6666666865348816,
+                    0.6000000238418579,
+                    0.5,
+                    0.5,
+                    0.6666666865348816,
+                    0.6666666865348816,
+                    0.6666666865348816,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9866666793823242,
+                    0.9599999785423279,
+                    0.9466666579246521,
+                    0.9333333373069763,
+                    0.8933333158493042,
+                    0.8399999737739563,
+                    0.8133333325386047,
+                    0.7866666913032532,
+                    0.746666669845581,
+                    0.6933333277702332,
+                    0.6399999856948853,
+                    0.5199999809265137,
+                    0.46666666865348816,
+                    0.4399999976158142,
+                    0.41333332657814026,
+                    0.36000001430511475,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.2933333218097687,
+                    0.2800000011920929,
+                    0.2800000011920929,
+                    0.20000000298023224,
+                    0.1733333319425583,
+                    0.13333334028720856,
+                    0.1066666692495346,
+                    0.06666667014360428,
+                    0.06666667014360428,
+                    0.0533333346247673,
+                    0.03999999910593033,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0.013333333656191826,
+                    0.013333333656191826
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    74,
+                    72,
+                    71,
+                    70,
+                    67,
+                    63,
+                    61,
+                    59,
+                    56,
+                    52,
+                    48,
+                    39,
+                    35,
+                    33,
+                    31,
+                    27,
+                    25,
+                    25,
+                    22,
+                    21,
+                    21,
+                    15,
+                    13,
+                    10,
+                    8,
+                    5,
+                    5,
+                    4,
+                    3,
+                    2,
+                    2,
+                    2,
+                    2,
+                    2,
+                    2,
+                    2,
+                    2,
+                    1,
+                    1
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    146,
+                    140,
+                    136,
+                    131,
+                    121,
+                    110,
+                    98,
+                    87,
+                    79,
+                    66,
+                    56,
+                    52,
+                    43,
+                    36,
+                    34,
+                    31,
+                    26,
+                    21,
+                    20,
+                    19,
+                    14,
+                    12,
+                    11,
+                    9,
+                    9,
+                    9,
+                    6,
+                    2,
+                    2,
+                    2,
+                    2,
+                    1,
+                    1,
+                    1,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    4,
+                    10,
+                    14,
+                    19,
+                    29,
+                    40,
+                    52,
+                    63,
+                    71,
+                    84,
+                    94,
+                    98,
+                    107,
+                    114,
+                    116,
+                    119,
+                    124,
+                    129,
+                    130,
+                    131,
+                    136,
+                    138,
+                    139,
+                    141,
+                    141,
+                    141,
+                    144,
+                    148,
+                    148,
+                    148,
+                    148,
+                    149,
+                    149,
+                    149,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    3,
+                    4,
+                    5,
+                    8,
+                    12,
+                    14,
+                    16,
+                    19,
+                    23,
+                    27,
+                    36,
+                    40,
+                    42,
+                    44,
+                    48,
+                    50,
+                    50,
+                    53,
+                    54,
+                    54,
+                    60,
+                    62,
+                    65,
+                    67,
+                    70,
+                    70,
+                    71,
+                    72,
+                    73,
+                    73,
+                    73,
+                    73,
+                    73,
+                    73,
+                    73,
+                    73,
+                    74,
+                    74
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82,
+                    0.84,
+                    0.86,
+                    0.88,
+                    0.9
+                ]
+            ],
+            [
+                1593069993791.473,
+                7,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3348214328289032,
+                    0.34246575832366943,
+                    0.3461538553237915,
+                    0.35567009449005127,
+                    0.36021506786346436,
+                    0.3735632300376892,
+                    0.3827160596847534,
+                    0.4054054021835327,
+                    0.3955223858356476,
+                    0.39830508828163147,
+                    0.4134615361690521,
+                    0.4204545319080353,
+                    0.44871795177459717,
+                    0.47887325286865234,
+                    0.5,
+                    0.5,
+                    0.5,
+                    0.5106382966041565,
+                    0.5365853905677795,
+                    0.5428571701049805,
+                    0.5625,
+                    0.5714285969734192,
+                    0.5769230723381042,
+                    0.5600000023841858,
+                    0.5454545617103577,
+                    0.5,
+                    0.5,
+                    0.5,
+                    0.4285714328289032,
+                    0.4545454680919647,
+                    0.4000000059604645,
+                    0.5,
+                    0.4000000059604645,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9599999785423279,
+                    0.9200000166893005,
+                    0.8933333158493042,
+                    0.8666666746139526,
+                    0.8266666531562805,
+                    0.800000011920929,
+                    0.7066666483879089,
+                    0.6266666650772095,
+                    0.5733333230018616,
+                    0.4933333396911621,
+                    0.46666666865348816,
+                    0.4533333480358124,
+                    0.4266666769981384,
+                    0.3733333349227905,
+                    0.3466666638851166,
+                    0.3199999928474426,
+                    0.2933333218097687,
+                    0.25333333015441895,
+                    0.23999999463558197,
+                    0.2133333384990692,
+                    0.20000000298023224,
+                    0.18666666746139526,
+                    0.1599999964237213,
+                    0.13333334028720856,
+                    0.11999999731779099,
+                    0.1066666692495346,
+                    0.07999999821186066,
+                    0.06666667014360428,
+                    0.0533333346247673,
+                    0.0533333346247673,
+                    0.02666666731238365,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    72,
+                    69,
+                    67,
+                    65,
+                    62,
+                    60,
+                    53,
+                    47,
+                    43,
+                    37,
+                    35,
+                    34,
+                    32,
+                    28,
+                    26,
+                    24,
+                    22,
+                    19,
+                    18,
+                    16,
+                    15,
+                    14,
+                    12,
+                    10,
+                    9,
+                    8,
+                    6,
+                    5,
+                    4,
+                    4,
+                    2,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    149,
+                    144,
+                    136,
+                    125,
+                    119,
+                    109,
+                    100,
+                    88,
+                    81,
+                    71,
+                    61,
+                    51,
+                    43,
+                    37,
+                    32,
+                    28,
+                    26,
+                    23,
+                    19,
+                    16,
+                    14,
+                    12,
+                    11,
+                    11,
+                    10,
+                    10,
+                    9,
+                    8,
+                    8,
+                    6,
+                    6,
+                    4,
+                    3,
+                    2,
+                    2,
+                    1,
+                    1,
+                    1,
+                    1
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    6,
+                    14,
+                    25,
+                    31,
+                    41,
+                    50,
+                    62,
+                    69,
+                    79,
+                    89,
+                    99,
+                    107,
+                    113,
+                    118,
+                    122,
+                    124,
+                    127,
+                    131,
+                    134,
+                    136,
+                    138,
+                    139,
+                    139,
+                    140,
+                    140,
+                    141,
+                    142,
+                    142,
+                    144,
+                    144,
+                    146,
+                    147,
+                    148,
+                    148,
+                    149,
+                    149,
+                    149,
+                    149
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    3,
+                    6,
+                    8,
+                    10,
+                    13,
+                    15,
+                    22,
+                    28,
+                    32,
+                    38,
+                    40,
+                    41,
+                    43,
+                    47,
+                    49,
+                    51,
+                    53,
+                    56,
+                    57,
+                    59,
+                    60,
+                    61,
+                    63,
+                    65,
+                    66,
+                    67,
+                    69,
+                    70,
+                    71,
+                    71,
+                    73,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82,
+                    0.84,
+                    0.86,
+                    0.88
+                ]
+            ],
+            [
+                1593069993792.149,
+                8,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3348214328289032,
+                    0.3363228738307953,
+                    0.3333333432674408,
+                    0.3396226465702057,
+                    0.3383084535598755,
+                    0.33673468232154846,
+                    0.3368983864784241,
+                    0.3314606845378876,
+                    0.33136093616485596,
+                    0.3561643958091736,
+                    0.3643410801887512,
+                    0.39316239953041077,
+                    0.43689319491386414,
+                    0.450549453496933,
+                    0.4938271641731262,
+                    0.4861111044883728,
+                    0.484375,
+                    0.5370370149612427,
+                    0.5333333611488342,
+                    0.5,
+                    0.5135135054588318,
+                    0.529411792755127,
+                    0.5806451439857483,
+                    0.6296296119689941,
+                    0.6499999761581421,
+                    0.6111111044883728,
+                    0.6666666865348816,
+                    0.7692307829856873,
+                    0.75,
+                    0.800000011920929,
+                    0.8333333134651184,
+                    0.800000011920929,
+                    0.800000011920929,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9733333587646484,
+                    0.9599999785423279,
+                    0.9066666960716248,
+                    0.8799999952316284,
+                    0.8399999737739563,
+                    0.7866666913032532,
+                    0.746666669845581,
+                    0.6933333277702332,
+                    0.6266666650772095,
+                    0.6133333444595337,
+                    0.6000000238418579,
+                    0.54666668176651,
+                    0.5333333611488342,
+                    0.46666666865348816,
+                    0.41333332657814026,
+                    0.3866666555404663,
+                    0.3199999928474426,
+                    0.25333333015441895,
+                    0.25333333015441895,
+                    0.23999999463558197,
+                    0.23999999463558197,
+                    0.2266666740179062,
+                    0.1733333319425583,
+                    0.14666666090488434,
+                    0.13333334028720856,
+                    0.13333334028720856,
+                    0.11999999731779099,
+                    0.1066666692495346,
+                    0.06666667014360428,
+                    0.0533333346247673,
+                    0.0533333346247673,
+                    0.0533333346247673,
+                    0.03999999910593033,
+                    0.03999999910593033,
+                    0.03999999910593033,
+                    0.013333333656191826,
+                    0.013333333656191826
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    73,
+                    72,
+                    68,
+                    66,
+                    63,
+                    59,
+                    56,
+                    52,
+                    47,
+                    46,
+                    45,
+                    41,
+                    40,
+                    35,
+                    31,
+                    29,
+                    24,
+                    19,
+                    19,
+                    18,
+                    18,
+                    17,
+                    13,
+                    11,
+                    10,
+                    10,
+                    9,
+                    8,
+                    5,
+                    4,
+                    4,
+                    4,
+                    3,
+                    3,
+                    3,
+                    1,
+                    1
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    149,
+                    148,
+                    146,
+                    140,
+                    133,
+                    130,
+                    124,
+                    119,
+                    113,
+                    94,
+                    82,
+                    71,
+                    58,
+                    50,
+                    41,
+                    37,
+                    33,
+                    25,
+                    21,
+                    19,
+                    18,
+                    16,
+                    13,
+                    10,
+                    7,
+                    7,
+                    5,
+                    3,
+                    3,
+                    2,
+                    1,
+                    1,
+                    1,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    1,
+                    2,
+                    4,
+                    10,
+                    17,
+                    20,
+                    26,
+                    31,
+                    37,
+                    56,
+                    68,
+                    79,
+                    92,
+                    100,
+                    109,
+                    113,
+                    117,
+                    125,
+                    129,
+                    131,
+                    132,
+                    134,
+                    137,
+                    140,
+                    143,
+                    143,
+                    145,
+                    147,
+                    147,
+                    148,
+                    149,
+                    149,
+                    149,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    150
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    2,
+                    3,
+                    7,
+                    9,
+                    12,
+                    16,
+                    19,
+                    23,
+                    28,
+                    29,
+                    30,
+                    34,
+                    35,
+                    40,
+                    44,
+                    46,
+                    51,
+                    56,
+                    56,
+                    57,
+                    57,
+                    58,
+                    62,
+                    64,
+                    65,
+                    65,
+                    66,
+                    67,
+                    70,
+                    71,
+                    71,
+                    71,
+                    72,
+                    72,
+                    72,
+                    74,
+                    74
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82,
+                    0.84,
+                    0.86
+                ]
+            ],
+            [
+                1593069993792.763,
+                9,
+                [
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3333333432674408,
+                    0.3363228738307953,
+                    0.3395348787307739,
+                    0.3349282443523407,
+                    0.3283582031726837,
+                    0.32474225759506226,
+                    0.34682080149650574,
+                    0.3522012531757355,
+                    0.3611111044883728,
+                    0.36567163467407227,
+                    0.3781512677669525,
+                    0.37962964177131653,
+                    0.3958333432674408,
+                    0.42352941632270813,
+                    0.4399999976158142,
+                    0.4637681245803833,
+                    0.4754098355770111,
+                    0.46296295523643494,
+                    0.47058823704719543,
+                    0.4791666567325592,
+                    0.5128205418586731,
+                    0.5142857432365417,
+                    0.5161290168762207,
+                    0.5199999809265137,
+                    0.6000000238418579,
+                    0.5555555820465088,
+                    0.6666666865348816,
+                    0.7777777910232544,
+                    0.8571428656578064,
+                    0.8571428656578064,
+                    0.8571428656578064,
+                    0.8571428656578064,
+                    0.8333333134651184,
+                    0.8333333134651184,
+                    0.6666666865348816,
+                    0.6666666865348816,
+                    0
+                ],
+                [
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    0.9733333587646484,
+                    0.9333333373069763,
+                    0.8799999952316284,
+                    0.8399999737739563,
+                    0.800000011920929,
+                    0.746666669845581,
+                    0.6933333277702332,
+                    0.653333306312561,
+                    0.6000000238418579,
+                    0.54666668176651,
+                    0.5066666603088379,
+                    0.47999998927116394,
+                    0.4399999976158142,
+                    0.4266666769981384,
+                    0.3866666555404663,
+                    0.3333333432674408,
+                    0.3199999928474426,
+                    0.30666667222976685,
+                    0.2666666805744171,
+                    0.23999999463558197,
+                    0.2133333384990692,
+                    0.1733333319425583,
+                    0.1599999964237213,
+                    0.13333334028720856,
+                    0.13333334028720856,
+                    0.09333333373069763,
+                    0.07999999821186066,
+                    0.07999999821186066,
+                    0.07999999821186066,
+                    0.07999999821186066,
+                    0.06666667014360428,
+                    0.06666667014360428,
+                    0.02666666731238365,
+                    0.02666666731238365,
+                    0
+                ],
+                [
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    75,
+                    73,
+                    70,
+                    66,
+                    63,
+                    60,
+                    56,
+                    52,
+                    49,
+                    45,
+                    41,
+                    38,
+                    36,
+                    33,
+                    32,
+                    29,
+                    25,
+                    24,
+                    23,
+                    20,
+                    18,
+                    16,
+                    13,
+                    12,
+                    10,
+                    10,
+                    7,
+                    6,
+                    6,
+                    6,
+                    6,
+                    5,
+                    5,
+                    2,
+                    2,
+                    0
+                ],
+                [
+                    150,
+                    150,
+                    150,
+                    150,
+                    150,
+                    148,
+                    142,
+                    139,
+                    135,
+                    131,
+                    113,
+                    103,
+                    92,
+                    85,
+                    74,
+                    67,
+                    58,
+                    49,
+                    42,
+                    37,
+                    32,
+                    29,
+                    27,
+                    25,
+                    19,
+                    17,
+                    15,
+                    12,
+                    8,
+                    8,
+                    5,
+                    2,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1,
+                    1
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    2,
+                    8,
+                    11,
+                    15,
+                    19,
+                    37,
+                    47,
+                    58,
+                    65,
+                    76,
+                    83,
+                    92,
+                    101,
+                    108,
+                    113,
+                    118,
+                    121,
+                    123,
+                    125,
+                    131,
+                    133,
+                    135,
+                    138,
+                    142,
+                    142,
+                    145,
+                    148,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149,
+                    149
+                ],
+                [
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    0,
+                    2,
+                    5,
+                    9,
+                    12,
+                    15,
+                    19,
+                    23,
+                    26,
+                    30,
+                    34,
+                    37,
+                    39,
+                    42,
+                    43,
+                    46,
+                    50,
+                    51,
+                    52,
+                    55,
+                    57,
+                    59,
+                    62,
+                    63,
+                    65,
+                    65,
+                    68,
+                    69,
+                    69,
+                    69,
+                    69,
+                    70,
+                    70,
+                    73,
+                    73,
+                    75
+                ],
+                [
+                    0.02,
+                    0.04,
+                    0.06,
+                    0.08,
+                    0.1,
+                    0.12,
+                    0.14,
+                    0.16,
+                    0.18,
+                    0.2,
+                    0.22,
+                    0.24,
+                    0.26,
+                    0.28,
+                    0.3,
+                    0.32,
+                    0.34,
+                    0.36,
+                    0.38,
+                    0.4,
+                    0.42,
+                    0.44,
+                    0.46,
+                    0.48,
+                    0.5,
+                    0.52,
+                    0.54,
+                    0.56,
+                    0.58,
+                    0.6,
+                    0.62,
+                    0.64,
+                    0.66,
+                    0.68,
+                    0.7,
+                    0.72,
+                    0.74,
+                    0.76,
+                    0.78,
+                    0.8,
+                    0.82
+                ]
+            ]
+        ];
+    }
+    return [
+        [
+            1593069993538.6739,
+            0,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3340757191181183,
+                0.3340807259082794,
+                0.3371298313140869,
+                0.3372093141078949,
+                0.35207822918891907,
+                0.36269429326057434,
+                0.3777777850627899,
+                0.3857566714286804,
+                0.39087948203086853,
+                0.4021352231502533,
+                0.4166666567325592,
+                0.42489269375801086,
+                0.44607841968536377,
+                0.4748603403568268,
+                0.4909090995788574,
+                0.5104895234107971,
+                0.48120301961898804,
+                0.5041322112083435,
+                0.5154638886451721,
+                0.517241358757019,
+                0.5263158082962036,
+                0.6140350699424744,
+                0.6595744490623474,
+                0.675000011920929,
+                0.6666666865348816,
+                0.7727272510528564,
+                0.7368420958518982,
+                0.7058823704719543,
+                0.7692307829856873,
+                0.800000011920929,
+                0.800000011920929,
+                0.7777777910232544,
+                0.75,
+                0.7142857313156128,
+                0.6000000238418579,
+                0.5,
+                0.5,
+                0.5,
+                0,
+                0,
+                0
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9933333396911621,
+                0.9866666793823242,
+                0.9666666388511658,
+                0.9599999785423279,
+                0.9333333373069763,
+                0.9066666960716248,
+                0.8666666746139526,
+                0.800000011920929,
+                0.753333330154419,
+                0.699999988079071,
+                0.6600000262260437,
+                0.6066666841506958,
+                0.5666666626930237,
+                0.5400000214576721,
+                0.4866666793823242,
+                0.4266666769981384,
+                0.40666666626930237,
+                0.3333333432674408,
+                0.30000001192092896,
+                0.2666666805744171,
+                0.23333333432674408,
+                0.20666666328907013,
+                0.18000000715255737,
+                0.13333334028720856,
+                0.1133333370089531,
+                0.09333333373069763,
+                0.07999999821186066,
+                0.06666667014360428,
+                0.0533333346247673,
+                0.0533333346247673,
+                0.046666666865348816,
+                0.03999999910593033,
+                0.03333333507180214,
+                0.019999999552965164,
+                0.013333333656191826,
+                0.006666666828095913,
+                0.006666666828095913,
+                0,
+                0,
+                0
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                149,
+                148,
+                145,
+                144,
+                140,
+                136,
+                130,
+                120,
+                113,
+                105,
+                99,
+                91,
+                85,
+                81,
+                73,
+                64,
+                61,
+                50,
+                45,
+                40,
+                35,
+                31,
+                27,
+                20,
+                17,
+                14,
+                12,
+                10,
+                8,
+                8,
+                7,
+                6,
+                5,
+                3,
+                2,
+                1,
+                1,
+                0,
+                0,
+                0
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                300,
+                299,
+                297,
+                291,
+                285,
+                265,
+                246,
+                224,
+                207,
+                187,
+                168,
+                147,
+                134,
+                113,
+                94,
+                84,
+                70,
+                69,
+                60,
+                47,
+                42,
+                36,
+                22,
+                16,
+                13,
+                10,
+                5,
+                5,
+                5,
+                3,
+                2,
+                2,
+                2,
+                2,
+                2,
+                2,
+                2,
+                1,
+                1,
+                1,
+                1,
+                1
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                3,
+                9,
+                15,
+                35,
+                54,
+                76,
+                93,
+                113,
+                132,
+                153,
+                166,
+                187,
+                206,
+                216,
+                230,
+                231,
+                240,
+                253,
+                258,
+                264,
+                278,
+                284,
+                287,
+                290,
+                295,
+                295,
+                295,
+                297,
+                298,
+                298,
+                298,
+                298,
+                298,
+                298,
+                298,
+                299,
+                299,
+                299,
+                299,
+                299
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                2,
+                5,
+                6,
+                10,
+                14,
+                20,
+                30,
+                37,
+                45,
+                51,
+                59,
+                65,
+                69,
+                77,
+                86,
+                89,
+                100,
+                105,
+                110,
+                115,
+                119,
+                123,
+                130,
+                133,
+                136,
+                138,
+                140,
+                142,
+                142,
+                143,
+                144,
+                145,
+                147,
+                148,
+                149,
+                149,
+                150,
+                150,
+                150
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82,
+                0.84,
+                0.86,
+                0.88,
+                0.9,
+                0.92
+            ]
+        ],
+        [
+            1593069993539.396,
+            1,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.33557048439979553,
+                0.33560091257095337,
+                0.3498817980289459,
+                0.35853657126426697,
+                0.37402597069740295,
+                0.3835616409778595,
+                0.3797101378440857,
+                0.3967213034629822,
+                0.40072202682495117,
+                0.40816327929496765,
+                0.4139534831047058,
+                0.4334975481033325,
+                0.45652174949645996,
+                0.4848484992980957,
+                0.4829931855201721,
+                0.5,
+                0.49180328845977783,
+                0.49074074625968933,
+                0.52173912525177,
+                0.5443037748336792,
+                0.5352112650871277,
+                0.5245901346206665,
+                0.4901960790157318,
+                0.4545454680919647,
+                0.4324324429035187,
+                0.4375,
+                0.4399999976158142,
+                0.5,
+                0.5625,
+                0.5,
+                0.4000000059604645,
+                0.5,
+                0.4000000059604645,
+                0.5,
+                0,
+                0,
+                0
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9866666793823242,
+                0.9866666793823242,
+                0.9800000190734863,
+                0.9599999785423279,
+                0.9333333373069763,
+                0.8733333349227905,
+                0.8066666722297668,
+                0.7400000095367432,
+                0.6666666865348816,
+                0.5933333039283752,
+                0.5866666436195374,
+                0.5600000023841858,
+                0.5333333611488342,
+                0.47333332896232605,
+                0.46000000834465027,
+                0.4000000059604645,
+                0.35333332419395447,
+                0.3199999928474426,
+                0.2866666615009308,
+                0.25333333015441895,
+                0.2133333384990692,
+                0.1666666716337204,
+                0.13333334028720856,
+                0.1066666692495346,
+                0.09333333373069763,
+                0.07333333045244217,
+                0.06666667014360428,
+                0.05999999865889549,
+                0.03999999910593033,
+                0.02666666731238365,
+                0.02666666731238365,
+                0.013333333656191826,
+                0.006666666828095913,
+                0,
+                0,
+                0
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                148,
+                148,
+                147,
+                144,
+                140,
+                131,
+                121,
+                111,
+                100,
+                89,
+                88,
+                84,
+                80,
+                71,
+                69,
+                60,
+                53,
+                48,
+                43,
+                38,
+                32,
+                25,
+                20,
+                16,
+                14,
+                11,
+                10,
+                9,
+                6,
+                4,
+                4,
+                2,
+                1,
+                0,
+                0,
+                0
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                300,
+                300,
+                297,
+                293,
+                275,
+                263,
+                241,
+                225,
+                214,
+                184,
+                166,
+                145,
+                126,
+                115,
+                100,
+                85,
+                76,
+                69,
+                62,
+                55,
+                44,
+                36,
+                33,
+                29,
+                26,
+                24,
+                21,
+                18,
+                14,
+                10,
+                7,
+                6,
+                6,
+                4,
+                3,
+                1,
+                1,
+                1,
+                1
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                3,
+                7,
+                25,
+                37,
+                59,
+                75,
+                86,
+                116,
+                134,
+                155,
+                174,
+                185,
+                200,
+                215,
+                224,
+                231,
+                238,
+                245,
+                256,
+                264,
+                267,
+                271,
+                274,
+                276,
+                279,
+                282,
+                286,
+                290,
+                293,
+                294,
+                294,
+                296,
+                297,
+                299,
+                299,
+                299,
+                299
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                2,
+                2,
+                3,
+                6,
+                10,
+                19,
+                29,
+                39,
+                50,
+                61,
+                62,
+                66,
+                70,
+                79,
+                81,
+                90,
+                97,
+                102,
+                107,
+                112,
+                118,
+                125,
+                130,
+                134,
+                136,
+                139,
+                140,
+                141,
+                144,
+                146,
+                146,
+                148,
+                149,
+                150,
+                150,
+                150
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82,
+                0.84,
+                0.86
+            ]
+        ],
+        [
+            1593069993540.066,
+            2,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3348214328289032,
+                0.33484163880348206,
+                0.33870968222618103,
+                0.34433960914611816,
+                0.35749998688697815,
+                0.3658536672592163,
+                0.3742690086364746,
+                0.3853503167629242,
+                0.4027777910232544,
+                0.40784314274787903,
+                0.4025973975658417,
+                0.4166666567325592,
+                0.42328041791915894,
+                0.45783132314682007,
+                0.4751773178577423,
+                0.4296875,
+                0.4568965435028076,
+                0.4476190507411957,
+                0.4838709533214569,
+                0.5121951103210449,
+                0.4931506812572479,
+                0.508474588394165,
+                0.4901960790157318,
+                0.4888888895511627,
+                0.5555555820465088,
+                0.6333333253860474,
+                0.6296296119689941,
+                0.75,
+                0.75,
+                0.6666666865348816,
+                0.75,
+                0.7142857313156128,
+                0.6666666865348816,
+                0.3333333432674408,
+                0,
+                0
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9866666793823242,
+                0.9800000190734863,
+                0.9733333587646484,
+                0.95333331823349,
+                0.8999999761581421,
+                0.8533333539962769,
+                0.8066666722297668,
+                0.7733333110809326,
+                0.6933333277702332,
+                0.6200000047683716,
+                0.5666666626930237,
+                0.5333333611488342,
+                0.5066666603088379,
+                0.4466666579246521,
+                0.36666667461395264,
+                0.35333332419395447,
+                0.31333333253860474,
+                0.30000001192092896,
+                0.2800000011920929,
+                0.23999999463558197,
+                0.20000000298023224,
+                0.1666666716337204,
+                0.14666666090488434,
+                0.13333334028720856,
+                0.12666666507720947,
+                0.1133333370089531,
+                0.10000000149011612,
+                0.07999999821186066,
+                0.0533333346247673,
+                0.03999999910593033,
+                0.03333333507180214,
+                0.02666666731238365,
+                0.006666666828095913,
+                0,
+                0
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                148,
+                147,
+                146,
+                143,
+                135,
+                128,
+                121,
+                116,
+                104,
+                93,
+                85,
+                80,
+                76,
+                67,
+                55,
+                53,
+                47,
+                45,
+                42,
+                36,
+                30,
+                25,
+                22,
+                20,
+                19,
+                17,
+                15,
+                12,
+                8,
+                6,
+                5,
+                4,
+                1,
+                0,
+                0
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                300,
+                298,
+                294,
+                287,
+                278,
+                257,
+                234,
+                214,
+                193,
+                172,
+                151,
+                138,
+                119,
+                109,
+                90,
+                74,
+                73,
+                63,
+                58,
+                48,
+                40,
+                37,
+                29,
+                26,
+                23,
+                16,
+                11,
+                10,
+                5,
+                4,
+                4,
+                2,
+                2,
+                2,
+                2,
+                1,
+                1
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                2,
+                6,
+                13,
+                22,
+                43,
+                66,
+                86,
+                107,
+                128,
+                149,
+                162,
+                181,
+                191,
+                210,
+                226,
+                227,
+                237,
+                242,
+                252,
+                260,
+                263,
+                271,
+                274,
+                277,
+                284,
+                289,
+                290,
+                295,
+                296,
+                296,
+                298,
+                298,
+                298,
+                298,
+                299,
+                299
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                2,
+                3,
+                4,
+                7,
+                15,
+                22,
+                29,
+                34,
+                46,
+                57,
+                65,
+                70,
+                74,
+                83,
+                95,
+                97,
+                103,
+                105,
+                108,
+                114,
+                120,
+                125,
+                128,
+                130,
+                131,
+                133,
+                135,
+                138,
+                142,
+                144,
+                145,
+                146,
+                149,
+                150,
+                150
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82
+            ]
+        ],
+        [
+            1593069993540.662,
+            3,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.33178654313087463,
+                0.33571428060531616,
+                0.3350125849246979,
+                0.3448275923728943,
+                0.36467236280441284,
+                0.3639143705368042,
+                0.3877550959587097,
+                0.38661709427833557,
+                0.38211381435394287,
+                0.3963963985443115,
+                0.4108910858631134,
+                0.43406593799591064,
+                0.4491018056869507,
+                0.4652777910232544,
+                0.4880000054836273,
+                0.4955752193927765,
+                0.5145630836486816,
+                0.5,
+                0.5057471394538879,
+                0.5194805264472961,
+                0.5373134613037109,
+                0.5789473652839661,
+                0.6000000238418579,
+                0.5952380895614624,
+                0.5714285969734192,
+                0.6785714030265808,
+                0.6521739363670349,
+                0.6666666865348816,
+                0.7142857313156128,
+                0.7272727489471436,
+                0.6666666865348816,
+                0.5,
+                0.6000000238418579,
+                0.5,
+                0.6666666865348816,
+                0.6666666865348816,
+                0.6666666865348816,
+                0.5,
+                1
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9866666793823242,
+                0.95333331823349,
+                0.9399999976158142,
+                0.8866666555404663,
+                0.8666666746139526,
+                0.8533333539962769,
+                0.7933333516120911,
+                0.7599999904632568,
+                0.6933333277702332,
+                0.6266666650772095,
+                0.5866666436195374,
+                0.5533333420753479,
+                0.5266666412353516,
+                0.5,
+                0.4466666579246521,
+                0.40666666626930237,
+                0.3733333349227905,
+                0.35333332419395447,
+                0.31333333253860474,
+                0.2933333218097687,
+                0.2666666805744171,
+                0.23999999463558197,
+                0.2199999988079071,
+                0.20000000298023224,
+                0.1666666716337204,
+                0.13333334028720856,
+                0.12666666507720947,
+                0.10000000149011612,
+                0.07999999821186066,
+                0.06666667014360428,
+                0.0533333346247673,
+                0.03999999910593033,
+                0.019999999552965164,
+                0.019999999552965164,
+                0.013333333656191826,
+                0.013333333656191826,
+                0.013333333656191826,
+                0.013333333656191826,
+                0.006666666828095913,
+                0.006666666828095913
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                148,
+                143,
+                141,
+                133,
+                130,
+                128,
+                119,
+                114,
+                104,
+                94,
+                88,
+                83,
+                79,
+                75,
+                67,
+                61,
+                56,
+                53,
+                47,
+                44,
+                40,
+                36,
+                33,
+                30,
+                25,
+                20,
+                19,
+                15,
+                12,
+                10,
+                8,
+                6,
+                3,
+                3,
+                2,
+                2,
+                2,
+                2,
+                1,
+                1
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                300,
+                300,
+                296,
+                288,
+                279,
+                264,
+                247,
+                223,
+                208,
+                180,
+                165,
+                152,
+                134,
+                119,
+                103,
+                92,
+                77,
+                64,
+                57,
+                50,
+                47,
+                43,
+                37,
+                31,
+                24,
+                20,
+                17,
+                15,
+                9,
+                8,
+                6,
+                4,
+                3,
+                3,
+                3,
+                2,
+                2,
+                1,
+                1,
+                1,
+                1,
+                0
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                4,
+                12,
+                21,
+                36,
+                53,
+                77,
+                92,
+                120,
+                135,
+                148,
+                166,
+                181,
+                197,
+                208,
+                223,
+                236,
+                243,
+                250,
+                253,
+                257,
+                263,
+                269,
+                276,
+                280,
+                283,
+                285,
+                291,
+                292,
+                294,
+                296,
+                297,
+                297,
+                297,
+                298,
+                298,
+                299,
+                299,
+                299,
+                299,
+                300
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                2,
+                7,
+                9,
+                17,
+                20,
+                22,
+                31,
+                36,
+                46,
+                56,
+                62,
+                67,
+                71,
+                75,
+                83,
+                89,
+                94,
+                97,
+                103,
+                106,
+                110,
+                114,
+                117,
+                120,
+                125,
+                130,
+                131,
+                135,
+                138,
+                140,
+                142,
+                144,
+                147,
+                147,
+                148,
+                148,
+                148,
+                148,
+                149,
+                149
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82,
+                0.84,
+                0.86,
+                0.88,
+                0.9,
+                0.92
+            ]
+        ],
+        [
+            1593069993541.333,
+            4,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3340757191181183,
+                0.3363228738307953,
+                0.33563217520713806,
+                0.3445783257484436,
+                0.3544303774833679,
+                0.36164382100105286,
+                0.366568922996521,
+                0.369905948638916,
+                0.37931033968925476,
+                0.3807692229747772,
+                0.40611353516578674,
+                0.41747573018074036,
+                0.44324323534965515,
+                0.4522292912006378,
+                0.48905110359191895,
+                0.4880000054836273,
+                0.5185185074806213,
+                0.5257731676101685,
+                0.5454545617103577,
+                0.5512820482254028,
+                0.5735294222831726,
+                0.5964912176132202,
+                0.5961538553237915,
+                0.5609756112098694,
+                0.5675675868988037,
+                0.6428571343421936,
+                0.6363636255264282,
+                0.6470588445663452,
+                0.5,
+                0.5454545617103577,
+                0.5555555820465088,
+                0.625,
+                0.6666666865348816,
+                0.3333333432674408,
+                0,
+                0,
+                0,
+                0
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9733333587646484,
+                0.95333331823349,
+                0.9333333373069763,
+                0.8799999952316284,
+                0.8333333134651184,
+                0.7866666913032532,
+                0.7333333492279053,
+                0.6600000262260437,
+                0.6200000047683716,
+                0.5733333230018616,
+                0.54666668176651,
+                0.47333332896232605,
+                0.4466666579246521,
+                0.40666666626930237,
+                0.3733333349227905,
+                0.3400000035762787,
+                0.3199999928474426,
+                0.2866666615009308,
+                0.25999999046325684,
+                0.2266666740179062,
+                0.20666666328907013,
+                0.15333333611488342,
+                0.14000000059604645,
+                0.11999999731779099,
+                0.09333333373069763,
+                0.07333333045244217,
+                0.03999999910593033,
+                0.03999999910593033,
+                0.03333333507180214,
+                0.03333333507180214,
+                0.02666666731238365,
+                0.006666666828095913,
+                0,
+                0,
+                0,
+                0
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                146,
+                143,
+                140,
+                132,
+                125,
+                118,
+                110,
+                99,
+                93,
+                86,
+                82,
+                71,
+                67,
+                61,
+                56,
+                51,
+                48,
+                43,
+                39,
+                34,
+                31,
+                23,
+                21,
+                18,
+                14,
+                11,
+                6,
+                6,
+                5,
+                5,
+                4,
+                1,
+                0,
+                0,
+                0,
+                0
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                300,
+                299,
+                296,
+                289,
+                272,
+                255,
+                233,
+                216,
+                201,
+                180,
+                161,
+                136,
+                120,
+                103,
+                86,
+                70,
+                64,
+                52,
+                46,
+                40,
+                35,
+                29,
+                23,
+                21,
+                18,
+                16,
+                10,
+                8,
+                6,
+                6,
+                5,
+                4,
+                3,
+                2,
+                2,
+                2,
+                2,
+                1,
+                1
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                4,
+                11,
+                28,
+                45,
+                67,
+                84,
+                99,
+                120,
+                139,
+                164,
+                180,
+                197,
+                214,
+                230,
+                236,
+                248,
+                254,
+                260,
+                265,
+                271,
+                277,
+                279,
+                282,
+                284,
+                290,
+                292,
+                294,
+                294,
+                295,
+                296,
+                297,
+                298,
+                298,
+                298,
+                298,
+                299,
+                299
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                4,
+                7,
+                10,
+                18,
+                25,
+                32,
+                40,
+                51,
+                57,
+                64,
+                68,
+                79,
+                83,
+                89,
+                94,
+                99,
+                102,
+                107,
+                111,
+                116,
+                119,
+                127,
+                129,
+                132,
+                136,
+                139,
+                144,
+                144,
+                145,
+                145,
+                146,
+                149,
+                150,
+                150,
+                150,
+                150
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82,
+                0.84,
+                0.86
+            ]
+        ],
+        [
+            1593069993542.078,
+            5,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.33557048439979553,
+                0.3355855941772461,
+                0.33944955468177795,
+                0.3501199185848236,
+                0.35249999165534973,
+                0.35978835821151733,
+                0.35919541120529175,
+                0.35403725504875183,
+                0.37288135290145874,
+                0.3730769157409668,
+                0.3822222352027893,
+                0.39086294174194336,
+                0.42941176891326904,
+                0.4394904375076294,
+                0.47183099389076233,
+                0.5079365372657776,
+                0.5398229956626892,
+                0.5148515105247498,
+                0.5348837375640869,
+                0.5432098507881165,
+                0.5882353186607361,
+                0.5932203531265259,
+                0.5714285969734192,
+                0.5909090638160706,
+                0.6341463327407837,
+                0.6285714507102966,
+                0.6333333253860474,
+                0.6000000238418579,
+                0.5714285969734192,
+                0.5263158082962036,
+                0.5,
+                0.4166666567325592,
+                0.2222222238779068,
+                0.2857142984867096,
+                0.25,
+                0.3333333432674408,
+                0,
+                0,
+                0
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9933333396911621,
+                0.9866666793823242,
+                0.9733333587646484,
+                0.9399999976158142,
+                0.9066666960716248,
+                0.8333333134651184,
+                0.7599999904632568,
+                0.7333333492279053,
+                0.6466666460037231,
+                0.5733333230018616,
+                0.5133333206176758,
+                0.4866666793823242,
+                0.46000000834465027,
+                0.4466666579246521,
+                0.4266666769981384,
+                0.40666666626930237,
+                0.3466666638851166,
+                0.30666667222976685,
+                0.2933333218097687,
+                0.2666666805744171,
+                0.23333333432674408,
+                0.18666666746139526,
+                0.1733333319425583,
+                0.1733333319425583,
+                0.14666666090488434,
+                0.12666666507720947,
+                0.10000000149011612,
+                0.07999999821186066,
+                0.06666667014360428,
+                0.046666666865348816,
+                0.03333333507180214,
+                0.013333333656191826,
+                0.013333333656191826,
+                0.006666666828095913,
+                0.006666666828095913,
+                0,
+                0,
+                0
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                149,
+                148,
+                146,
+                141,
+                136,
+                125,
+                114,
+                110,
+                97,
+                86,
+                77,
+                73,
+                69,
+                67,
+                64,
+                61,
+                52,
+                46,
+                44,
+                40,
+                35,
+                28,
+                26,
+                26,
+                22,
+                19,
+                15,
+                12,
+                10,
+                7,
+                5,
+                2,
+                2,
+                1,
+                1,
+                0,
+                0,
+                0
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                300,
+                297,
+                295,
+                288,
+                271,
+                259,
+                242,
+                223,
+                208,
+                185,
+                163,
+                139,
+                120,
+                97,
+                88,
+                75,
+                62,
+                52,
+                49,
+                40,
+                37,
+                28,
+                24,
+                21,
+                18,
+                15,
+                13,
+                11,
+                10,
+                9,
+                9,
+                7,
+                7,
+                7,
+                5,
+                3,
+                2,
+                2,
+                1,
+                1
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                3,
+                5,
+                12,
+                29,
+                41,
+                58,
+                77,
+                92,
+                115,
+                137,
+                161,
+                180,
+                203,
+                212,
+                225,
+                238,
+                248,
+                251,
+                260,
+                263,
+                272,
+                276,
+                279,
+                282,
+                285,
+                287,
+                289,
+                290,
+                291,
+                291,
+                293,
+                293,
+                293,
+                295,
+                297,
+                298,
+                298,
+                299,
+                299
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                2,
+                4,
+                9,
+                14,
+                25,
+                36,
+                40,
+                53,
+                64,
+                73,
+                77,
+                81,
+                83,
+                86,
+                89,
+                98,
+                104,
+                106,
+                110,
+                115,
+                122,
+                124,
+                124,
+                128,
+                131,
+                135,
+                138,
+                140,
+                143,
+                145,
+                148,
+                148,
+                149,
+                149,
+                150,
+                150,
+                150
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82,
+                0.84,
+                0.86,
+                0.88
+            ]
+        ],
+        [
+            1593069993543.1821,
+            6,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3303571343421936,
+                0.33257919549942017,
+                0.33642691373825073,
+                0.34285715222358704,
+                0.35087719559669495,
+                0.36239781975746155,
+                0.37134504318237305,
+                0.38485804200172424,
+                0.4000000059604645,
+                0.4089219272136688,
+                0.43096235394477844,
+                0.43518519401550293,
+                0.4202127754688263,
+                0.42603549361228943,
+                0.43918919563293457,
+                0.44525548815727234,
+                0.45967742800712585,
+                0.4821428656578064,
+                0.5102040767669678,
+                0.529411792755127,
+                0.5128205418586731,
+                0.5797101259231567,
+                0.550000011920929,
+                0.5094339847564697,
+                0.4888888895511627,
+                0.46341463923454285,
+                0.375,
+                0.3461538553237915,
+                0.4000000059604645,
+                0.4000000059604645,
+                0.3333333432674408,
+                0.27272728085517883,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.2857142984867096,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.6666666865348816,
+                0.5,
+                1
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9866666793823242,
+                0.9800000190734863,
+                0.9666666388511658,
+                0.9599999785423279,
+                0.9333333373069763,
+                0.8866666555404663,
+                0.846666693687439,
+                0.8133333325386047,
+                0.7599999904632568,
+                0.7333333492279053,
+                0.6866666674613953,
+                0.6266666650772095,
+                0.5266666412353516,
+                0.47999998927116394,
+                0.4333333373069763,
+                0.40666666626930237,
+                0.3799999952316284,
+                0.36000001430511475,
+                0.3333333432674408,
+                0.30000001192092896,
+                0.2666666805744171,
+                0.2666666805744171,
+                0.2199999988079071,
+                0.18000000715255737,
+                0.14666666090488434,
+                0.12666666507720947,
+                0.07999999821186066,
+                0.05999999865889549,
+                0.0533333346247673,
+                0.03999999910593033,
+                0.02666666731238365,
+                0.019999999552965164,
+                0.019999999552965164,
+                0.019999999552965164,
+                0.013333333656191826,
+                0.013333333656191826,
+                0.013333333656191826,
+                0.013333333656191826,
+                0.006666666828095913,
+                0.006666666828095913
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                148,
+                147,
+                145,
+                144,
+                140,
+                133,
+                127,
+                122,
+                114,
+                110,
+                103,
+                94,
+                79,
+                72,
+                65,
+                61,
+                57,
+                54,
+                50,
+                45,
+                40,
+                40,
+                33,
+                27,
+                22,
+                19,
+                12,
+                9,
+                8,
+                6,
+                4,
+                3,
+                3,
+                3,
+                2,
+                2,
+                2,
+                2,
+                1,
+                1
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                300,
+                300,
+                295,
+                286,
+                276,
+                259,
+                234,
+                215,
+                195,
+                171,
+                159,
+                136,
+                122,
+                109,
+                97,
+                83,
+                76,
+                67,
+                58,
+                48,
+                40,
+                38,
+                29,
+                27,
+                26,
+                23,
+                22,
+                20,
+                17,
+                12,
+                9,
+                8,
+                8,
+                6,
+                6,
+                5,
+                4,
+                4,
+                1,
+                1,
+                0
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                5,
+                14,
+                24,
+                41,
+                66,
+                85,
+                105,
+                129,
+                141,
+                164,
+                178,
+                191,
+                203,
+                217,
+                224,
+                233,
+                242,
+                252,
+                260,
+                262,
+                271,
+                273,
+                274,
+                277,
+                278,
+                280,
+                283,
+                288,
+                291,
+                292,
+                292,
+                294,
+                294,
+                295,
+                296,
+                296,
+                299,
+                299,
+                300
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                2,
+                3,
+                5,
+                6,
+                10,
+                17,
+                23,
+                28,
+                36,
+                40,
+                47,
+                56,
+                71,
+                78,
+                85,
+                89,
+                93,
+                96,
+                100,
+                105,
+                110,
+                110,
+                117,
+                123,
+                128,
+                131,
+                138,
+                141,
+                142,
+                144,
+                146,
+                147,
+                147,
+                147,
+                148,
+                148,
+                148,
+                148,
+                149,
+                149
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82,
+                0.84,
+                0.86,
+                0.88,
+                0.9
+            ]
+        ],
+        [
+            1593069993543.998,
+            7,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3340757191181183,
+                0.3348214328289032,
+                0.33636364340782166,
+                0.3380281627178192,
+                0.3483709394931793,
+                0.35449734330177307,
+                0.3672316372394562,
+                0.381538450717926,
+                0.39464882016181946,
+                0.3970588147640228,
+                0.40416666865348816,
+                0.4084506928920746,
+                0.42328041791915894,
+                0.4497041404247284,
+                0.4868420958518982,
+                0.5072463750839233,
+                0.529411792755127,
+                0.5267857313156128,
+                0.5050504803657532,
+                0.5056179761886597,
+                0.5131579041481018,
+                0.5072463750839233,
+                0.515625,
+                0.5636363625526428,
+                0.5686274766921997,
+                0.5744680762290955,
+                0.5581395626068115,
+                0.5277777910232544,
+                0.5,
+                0.43478259444236755,
+                0.4444444477558136,
+                0.4375,
+                0.5833333134651184,
+                0.5,
+                0.25,
+                0,
+                0,
+                0,
+                0,
+                0
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9866666793823242,
+                0.9599999785423279,
+                0.9266666769981384,
+                0.8933333158493042,
+                0.8666666746139526,
+                0.8266666531562805,
+                0.7866666913032532,
+                0.7200000286102295,
+                0.6466666460037231,
+                0.5799999833106995,
+                0.5333333611488342,
+                0.5066666603088379,
+                0.4933333396911621,
+                0.46666666865348816,
+                0.41999998688697815,
+                0.3933333456516266,
+                0.3333333432674408,
+                0.30000001192092896,
+                0.25999999046325684,
+                0.23333333432674408,
+                0.2199999988079071,
+                0.20666666328907013,
+                0.19333332777023315,
+                0.18000000715255737,
+                0.1599999964237213,
+                0.12666666507720947,
+                0.10000000149011612,
+                0.06666667014360428,
+                0.0533333346247673,
+                0.046666666865348816,
+                0.046666666865348816,
+                0.02666666731238365,
+                0.006666666828095913,
+                0,
+                0,
+                0,
+                0,
+                0
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                150,
+                148,
+                144,
+                139,
+                134,
+                130,
+                124,
+                118,
+                108,
+                97,
+                87,
+                80,
+                76,
+                74,
+                70,
+                63,
+                59,
+                50,
+                45,
+                39,
+                35,
+                33,
+                31,
+                29,
+                27,
+                24,
+                19,
+                15,
+                10,
+                8,
+                7,
+                7,
+                4,
+                1,
+                0,
+                0,
+                0,
+                0,
+                0
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                299,
+                298,
+                292,
+                282,
+                260,
+                244,
+                224,
+                201,
+                181,
+                164,
+                143,
+                126,
+                109,
+                93,
+                78,
+                68,
+                56,
+                53,
+                49,
+                44,
+                37,
+                34,
+                31,
+                24,
+                22,
+                20,
+                19,
+                17,
+                15,
+                13,
+                10,
+                9,
+                5,
+                4,
+                3,
+                2,
+                1,
+                1,
+                1,
+                1
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                1,
+                2,
+                8,
+                18,
+                40,
+                56,
+                76,
+                99,
+                119,
+                136,
+                157,
+                174,
+                191,
+                207,
+                222,
+                232,
+                244,
+                247,
+                251,
+                256,
+                263,
+                266,
+                269,
+                276,
+                278,
+                280,
+                281,
+                283,
+                285,
+                287,
+                290,
+                291,
+                295,
+                296,
+                297,
+                298,
+                299,
+                299,
+                299,
+                299
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                2,
+                6,
+                11,
+                16,
+                20,
+                26,
+                32,
+                42,
+                53,
+                63,
+                70,
+                74,
+                76,
+                80,
+                87,
+                91,
+                100,
+                105,
+                111,
+                115,
+                117,
+                119,
+                121,
+                123,
+                126,
+                131,
+                135,
+                140,
+                142,
+                143,
+                143,
+                146,
+                149,
+                150,
+                150,
+                150,
+                150,
+                150
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82,
+                0.84,
+                0.86,
+                0.88
+            ]
+        ],
+        [
+            1593069993544.9128,
+            8,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3340757191181183,
+                0.33483144640922546,
+                0.33638444542884827,
+                0.3452380895614624,
+                0.34405940771102905,
+                0.3489583432674408,
+                0.35164836049079895,
+                0.34593021869659424,
+                0.35825544595718384,
+                0.38327527046203613,
+                0.3897637724876404,
+                0.4196428656578064,
+                0.44607841968536377,
+                0.45945945382118225,
+                0.4819277226924896,
+                0.4791666567325592,
+                0.484375,
+                0.5044247508049011,
+                0.5102040767669678,
+                0.5176470875740051,
+                0.5375000238418579,
+                0.5540540814399719,
+                0.5671641826629639,
+                0.5833333134651184,
+                0.6170212626457214,
+                0.625,
+                0.7096773982048035,
+                0.7307692170143127,
+                0.7727272510528564,
+                0.8333333134651184,
+                0.8571428656578064,
+                0.8181818127632141,
+                0.8999999761581421,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9933333396911621,
+                0.9800000190734863,
+                0.9666666388511658,
+                0.9266666769981384,
+                0.8933333158493042,
+                0.8533333539962769,
+                0.7933333516120911,
+                0.7666666507720947,
+                0.7333333492279053,
+                0.6600000262260437,
+                0.6266666650772095,
+                0.6066666841506958,
+                0.5666666626930237,
+                0.5333333611488342,
+                0.46000000834465027,
+                0.41333332657814026,
+                0.3799999952316284,
+                0.3333333432674408,
+                0.2933333218097687,
+                0.2866666615009308,
+                0.273333340883255,
+                0.25333333015441895,
+                0.23333333432674408,
+                0.19333332777023315,
+                0.1666666716337204,
+                0.14666666090488434,
+                0.12666666507720947,
+                0.1133333370089531,
+                0.10000000149011612,
+                0.07999999821186066,
+                0.05999999865889549,
+                0.05999999865889549,
+                0.046666666865348816,
+                0.03999999910593033,
+                0.03999999910593033,
+                0.03333333507180214,
+                0.013333333656191826,
+                0.013333333656191826
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                149,
+                147,
+                145,
+                139,
+                134,
+                128,
+                119,
+                115,
+                110,
+                99,
+                94,
+                91,
+                85,
+                80,
+                69,
+                62,
+                57,
+                50,
+                44,
+                43,
+                41,
+                38,
+                35,
+                29,
+                25,
+                22,
+                19,
+                17,
+                15,
+                12,
+                9,
+                9,
+                7,
+                6,
+                6,
+                5,
+                2,
+                2
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                299,
+                296,
+                290,
+                275,
+                265,
+                250,
+                236,
+                225,
+                206,
+                177,
+                155,
+                130,
+                113,
+                100,
+                86,
+                75,
+                66,
+                56,
+                48,
+                41,
+                37,
+                33,
+                29,
+                25,
+                18,
+                15,
+                9,
+                7,
+                5,
+                3,
+                2,
+                2,
+                1,
+                0,
+                0,
+                0,
+                0,
+                0,
+                0
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                1,
+                4,
+                10,
+                25,
+                35,
+                50,
+                64,
+                75,
+                94,
+                123,
+                145,
+                170,
+                187,
+                200,
+                214,
+                225,
+                234,
+                244,
+                252,
+                259,
+                263,
+                267,
+                271,
+                275,
+                282,
+                285,
+                291,
+                293,
+                295,
+                297,
+                298,
+                298,
+                299,
+                300,
+                300,
+                300,
+                300,
+                300,
+                300
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                3,
+                5,
+                11,
+                16,
+                22,
+                31,
+                35,
+                40,
+                51,
+                56,
+                59,
+                65,
+                70,
+                81,
+                88,
+                93,
+                100,
+                106,
+                107,
+                109,
+                112,
+                115,
+                121,
+                125,
+                128,
+                131,
+                133,
+                135,
+                138,
+                141,
+                141,
+                143,
+                144,
+                144,
+                145,
+                148,
+                148
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82,
+                0.84,
+                0.86
+            ]
+        ],
+        [
+            1593069993545.62,
+            9,
+            [
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3333333432674408,
+                0.3340807259082794,
+                0.33870968222618103,
+                0.3388625681400299,
+                0.3432098627090454,
+                0.35078534483909607,
+                0.37321937084198,
+                0.3792048990726471,
+                0.3872053921222687,
+                0.3955223858356476,
+                0.42016807198524475,
+                0.43317973613739014,
+                0.43617022037506104,
+                0.4588235318660736,
+                0.4736842215061188,
+                0.4848484992980957,
+                0.5,
+                0.49038460850715637,
+                0.5106382966041565,
+                0.5411764979362488,
+                0.5857142806053162,
+                0.5645161271095276,
+                0.5714285969734192,
+                0.6041666865348816,
+                0.6666666865348816,
+                0.6571428775787354,
+                0.7333333492279053,
+                0.8260869383811951,
+                0.8333333134651184,
+                0.875,
+                0.8181818127632141,
+                0.8181818127632141,
+                0.8888888955116272,
+                0.8888888955116272,
+                0.800000011920929,
+                0.800000011920929,
+                0
+            ],
+            [
+                1,
+                1,
+                1,
+                1,
+                1,
+                0.9933333396911621,
+                0.9800000190734863,
+                0.95333331823349,
+                0.9266666769981384,
+                0.8933333158493042,
+                0.8733333349227905,
+                0.8266666531562805,
+                0.7666666507720947,
+                0.7066666483879089,
+                0.6666666865348816,
+                0.6266666650772095,
+                0.54666668176651,
+                0.5199999809265137,
+                0.47999998927116394,
+                0.4266666769981384,
+                0.3933333456516266,
+                0.3400000035762787,
+                0.3199999928474426,
+                0.30666667222976685,
+                0.273333340883255,
+                0.23333333432674408,
+                0.2133333384990692,
+                0.19333332777023315,
+                0.1733333319425583,
+                0.15333333611488342,
+                0.14666666090488434,
+                0.12666666507720947,
+                0.10000000149011612,
+                0.09333333373069763,
+                0.05999999865889549,
+                0.05999999865889549,
+                0.0533333346247673,
+                0.0533333346247673,
+                0.02666666731238365,
+                0.02666666731238365,
+                0
+            ],
+            [
+                150,
+                150,
+                150,
+                150,
+                150,
+                149,
+                147,
+                143,
+                139,
+                134,
+                131,
+                124,
+                115,
+                106,
+                100,
+                94,
+                82,
+                78,
+                72,
+                64,
+                59,
+                51,
+                48,
+                46,
+                41,
+                35,
+                32,
+                29,
+                26,
+                23,
+                22,
+                19,
+                15,
+                14,
+                9,
+                9,
+                8,
+                8,
+                4,
+                4,
+                0
+            ],
+            [
+                300,
+                300,
+                300,
+                300,
+                300,
+                297,
+                287,
+                279,
+                266,
+                248,
+                220,
+                203,
+                182,
+                162,
+                138,
+                123,
+                106,
+                92,
+                80,
+                68,
+                59,
+                53,
+                46,
+                39,
+                29,
+                27,
+                24,
+                19,
+                13,
+                12,
+                8,
+                4,
+                3,
+                2,
+                2,
+                2,
+                1,
+                1,
+                1,
+                1,
+                1
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                3,
+                13,
+                21,
+                34,
+                52,
+                80,
+                97,
+                118,
+                138,
+                162,
+                177,
+                194,
+                208,
+                220,
+                232,
+                241,
+                247,
+                254,
+                261,
+                271,
+                273,
+                276,
+                281,
+                287,
+                288,
+                292,
+                296,
+                297,
+                298,
+                298,
+                298,
+                299,
+                299,
+                299,
+                299,
+                299
+            ],
+            [
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                3,
+                7,
+                11,
+                16,
+                19,
+                26,
+                35,
+                44,
+                50,
+                56,
+                68,
+                72,
+                78,
+                86,
+                91,
+                99,
+                102,
+                104,
+                109,
+                115,
+                118,
+                121,
+                124,
+                127,
+                128,
+                131,
+                135,
+                136,
+                141,
+                141,
+                142,
+                142,
+                146,
+                146,
+                150
+            ],
+            [
+                0.02,
+                0.04,
+                0.06,
+                0.08,
+                0.1,
+                0.12,
+                0.14,
+                0.16,
+                0.18,
+                0.2,
+                0.22,
+                0.24,
+                0.26,
+                0.28,
+                0.3,
+                0.32,
+                0.34,
+                0.36,
+                0.38,
+                0.4,
+                0.42,
+                0.44,
+                0.46,
+                0.48,
+                0.5,
+                0.52,
+                0.54,
+                0.56,
+                0.58,
+                0.6,
+                0.62,
+                0.64,
+                0.66,
+                0.68,
+                0.7,
+                0.72,
+                0.74,
+                0.76,
+                0.78,
+                0.8,
+                0.82
+            ]
+        ]
+    ];
+};

frontend/packages/mock/data/roc-curve/steps.ts

+/**
+ * Copyright 2020 Baidu Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Request} from 'express';
+
+export default (request: Request) => {
+    if (request.query.run === 'train') {
+        return [
+            [1593069993786.464, 0],
+            [1593069993787.353, 1],
+            [1593069993788.1448, 2],
+            [1593069993788.836, 3],
+            [1593069993789.4, 4],
+            [1593069993790.076, 5],
+            [1593069993790.763, 6],
+            [1593069993791.473, 7],
+            [1593069993792.149, 8],
+            [1593069993792.763, 9]
+        ];
+    }
+    return [
+        [1593069993538.6739, 0],
+        [1593069993539.396, 1],
+        [1593069993540.066, 2],
+        [1593069993540.662, 3],
+        [1593069993541.333, 4],
+        [1593069993542.078, 5],
+        [1593069993543.1821, 6],
+        [1593069993543.998, 7],
+        [1593069993544.9128, 8],
+        [1593069993545.62, 9]
+    ];
+};

frontend/packages/mock/data/roc-curve/tags.ts

+/**
+ * Copyright 2020 Baidu Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default {
+    runs: ['train', 'test'],
+    tags: [
+        ['layer2/biases/summaries/mean', 'test/1234', 'another'],
+        [
+            'layer2/biases/summaries/mean',
+            'layer2/biases/summaries/accuracy',
+            'layer2/biases/summaries/cost',
+            'test/431',
+            'others'
+        ]
+    ]
+};

frontend/packages/mock/data/scalar/data.ts

+/**
+ * Copyright 2020 Baidu Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Request, Response} from 'express';
+
+export default (req: Request, res: Response) => {
+    const {run, tag} = req.query;
+    res.setHeader('Content-Type', 'text/tab-separated-values');
+    return `scalar\n${run}\n${tag}`;
+};

frontend/yarn.lock

@@ -8231,6 +8231,11 @@ ignore@^5.1.4:
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57"
   integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==
 
+immediate@~3.0.5:
+  version "3.0.6"
+  resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
+  integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
+
 import-fresh@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-2.0.0.tgz#d81355c15612d386c61f9ddd3922d4304822a546"
@@ -9337,6 +9342,16 @@ jsprim@^1.2.2:
     array-includes "^3.1.1"
     object.assign "^4.1.1"
 
+jszip@3.5.0:
+  version "3.5.0"
+  resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.5.0.tgz#b4fd1f368245346658e781fec9675802489e15f6"
+  integrity sha512-WRtu7TPCmYePR1nazfrtuF216cIVon/3GWOvHS9QR5bIwSbnxtdpma6un3jyGGNhHsKCSzn5Ypk+EkDRvTGiFA==
+  dependencies:
+    lie "~3.3.0"
+    pako "~1.0.2"
+    readable-stream "~2.3.6"
+    set-immediate-shim "~1.0.1"
+
 junk@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/junk/-/junk-3.1.0.tgz#31499098d902b7e98c5d9b9c80f43457a88abfa1"
@@ -9464,6 +9479,13 @@ levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
+lie@~3.3.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
+  integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==
+  dependencies:
+    immediate "~3.0.5"
+
 lines-and-columns@^1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
@@ -11018,7 +11040,7 @@ package-json@^6.3.0:
     registry-url "^5.0.0"
     semver "^6.2.0"
 
-pako@1.0.11:
+pako@1.0.11, pako@~1.0.2:
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf"
   integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==
@@ -12683,6 +12705,11 @@ set-blocking@^2.0.0, set-blocking@~2.0.0:
   resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
   integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
 
+set-immediate-shim@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61"
+  integrity sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=
+
 set-value@^2.0.0, set-value@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b"

luckydraw.md

-# **抽奖规则**
-
- 
-
-### **方式一：点Star抽奖（20名）**
-
-•     **参与方式**：
-
-为[VisualDL](https://github.com/PaddlePaddle/VisualDL)或者 [PaddleX](https://github.com/PaddlePaddle/PaddleX)任意Repo 点星（Star）参与抽奖（也可双重点击累加，提高中奖几率）
-
-•     **奖项设置**：
-
-VDL Github抽取9名，PaddleX Github抽取9名（以下奖项均分）
-
-**一等奖（2名）** ：蓝牙键盘、飞桨充电宝
-
-**二等奖（2名）** ：价值50元的京东购物卡
-
-**三等奖（4名）** ：百度网盘超级会员
-
-**鼓励奖（10名）** ：飞桨鸭舌帽/飞桨精美帆布包 任选
-
-•     **领奖方式** ：
-
-根据开奖公告，获奖同学凭github账号截图到**VisualDL qq群（1045783368）@Yixin|Visualdl **小姐姐领奖
-
-
-
-### **方式二：在AI Studio上创建项目（3名）**
-
- 
-
-•     **参与方式**：
-
-在AI Studio上应用PaddleX 以及 VisualDL创建「计数」项目，并将项目链接回复至AI Studio的[VDL论坛评论区](https://ai.baidu.com/forum/topic/show/960053)：飞桨团队会对项目质量打分并选择得分最高的前三名送出奖品。
-
-•     **奖项设置**：蓝牙键盘*3
-
-•     **领奖方式** ：依依小姐姐将单独联系获奖的项目同学送出奖品
-
-  
-
-### **参与时间**：
-
-**9月21日--9月29日（下周二）**
-
-
-
-### **开奖方式**：
-
-**获奖名单将在9月30日公布于本页「0921直播抽奖活动」专栏**
-
-
-
-![Star示意](./imgs/star.png)
-
-![Star后ID呈现](./imgs/ID.jpg)
-
-
-# 获奖名单
-
-## 一等奖
-
-### VisualDL Repo
-
-恭喜用户**1084667371**获得**蓝牙键盘**一个！！
-
-### PaddleX Repo
-
-恭喜用户**Calvert97**获得**飞桨充电宝**一个！！
-
-
-
-## 二等奖
-
-### VisualDL Repo
-
-恭喜**0-yy-0**获得价值50元的**京东购物卡**！！
-
-### PaddleX Repo
-
-恭喜**AnkerLeng**获得价值50元的**京东购物卡**！！
-
-
-
-## 三等奖
-
-### VisualDL Repo
-
-恭喜**JetHong、gylidian**获得**百度网盘超级会员**！！
-
-### PaddleX Repo
-
-恭喜**loyasto、liu824**获得**百度网盘超级会员**！！
-
-
-
-## 阳光普照奖
-
-### VisualDL Repo
-
-恭喜**Sqhttwl、ckalong、dongtianqi1125、ralph0813、wjiawei97**获得**飞桨鸭舌帽 OR 飞桨帆布袋**！！
-
-### PaddleX Repo
-
-恭喜**wangluohaima、Yaoxingtian、rango42z、qiceng、mumucai**获得**飞桨鸭舌帽 OR 飞桨帆布袋**！！
-
-*注意：阳光普照奖获奖小伙伴们先领奖可以先选择心仪的奖品（飞桨鸭舌帽 OR 飞桨帆布袋）噢~~
-
-
-
-## 领奖方式
-
-**再次恭喜以上获奖小伙伴们，请加入VisualDL官方QQ群1045783368，在群里@依依小姐姐（Yixin|VisualDL）发送Github账号截图领奖~~**
-
-<p align="center">
-<img src="https://user-images.githubusercontent.com/48054808/82522691-c2758680-9b5c-11ea-9aee-fca994aba175.png" width="20%"/>
-</p>
-
-
- 
-

visualdl/component/__init__.py

@@ -34,6 +34,9 @@ components = {
     "pr_curve": {
         "enabled": False
     },
+    "roc_curve": {
+        "enabled": False
+    },
     "meta_data": {
         "enabled": False
     }

visualdl/component/base_component.py

@@ -404,3 +404,125 @@ def pr_curve_raw(tag, tp, fp, tn, fn, precision, recall, step, walltime):
         Record.Value(
             id=step, tag=tag, timestamp=walltime, pr_curve=prcurve)
     ])
+def compute_roc_curve(labels, predictions, num_thresholds=None, weights=None):
+    """ Compute ROC curve data by labels and predictions.
+    Args:
+        labels (numpy.ndarray or list): Binary labels for each element.
+        predictions (numpy.ndarray or list): The probability that an element be
+            classified as true.
+        num_thresholds (int): Number of thresholds used to draw the curve.
+        weights (float): Multiple of data to display on the curve.
+    """
+    if isinstance(labels, list):
+        labels = np.array(labels)
+    if isinstance(predictions, list):
+        predictions = np.array(predictions)
+    _MINIMUM_COUNT = 1e-7
+
+    if weights is None:
+        weights = 1.0
+
+    bucket_indices = np.int32(np.floor(predictions * (num_thresholds - 1)))
+    float_labels = labels.astype(np.float)
+    histogram_range = (0, num_thresholds - 1)
+    tp_buckets, _ = np.histogram(
+        bucket_indices,
+        bins=num_thresholds,
+        range=histogram_range,
+        weights=float_labels * weights)
+    fp_buckets, _ = np.histogram(
+        bucket_indices,
+        bins=num_thresholds,
+        range=histogram_range,
+        weights=(1.0 - float_labels) * weights)
+
+    # Obtain the reverse cumulative sum.
+    tp = np.cumsum(tp_buckets[::-1])[::-1]
+    fp = np.cumsum(fp_buckets[::-1])[::-1]
+    tn = fp[0] - fp
+    fn = tp[0] - tp
+    tpr = tp / np.maximum(_MINIMUM_COUNT, tn + fp)
+    fpr = fp / np.maximum(_MINIMUM_COUNT, tn + fp)
+    data = {
+        'tp': tp.astype(int).tolist(),
+        'fp': fp.astype(int).tolist(),
+        'tn': tn.astype(int).tolist(),
+        'fn': fn.astype(int).tolist(),
+        'tpr': tpr.astype(float).tolist(),
+        'fpr': fpr.astype(float).tolist()
+    }
+    return data
+
+
+def roc_curve(tag, labels, predictions, step, walltime, num_thresholds=127,
+             weights=None):
+    """Package data to one roc_curve.
+    Args:
+        tag (string): Data identifier
+        labels (numpy.ndarray or list): Binary labels for each element.
+        predictions (numpy.ndarray or list): The probability that an element be
+            classified as true.
+        step (int): Step of pr_curve
+        walltime (int): Wall time of pr_curve
+        num_thresholds (int): Number of thresholds used to draw the curve.
+        weights (float): Multiple of data to display on the curve.
+    Return:
+        Package with format of record_pb2.Record
+    """
+    num_thresholds = min(num_thresholds, 127)
+    roc_curve_map = compute_roc_curve(labels, predictions, num_thresholds, weights)
+
+    return roc_curve_raw(tag=tag,
+                        tp=roc_curve_map['tp'],
+                        fp=roc_curve_map['fp'],
+                        tn=roc_curve_map['tn'],
+                        fn=roc_curve_map['fn'],
+                        tpr=roc_curve_map['tpr'],
+                        fpr=roc_curve_map['fpr'],
+                        step=step,
+                        walltime=walltime)
+
+
+def roc_curve_raw(tag, tp, fp, tn, fn, tpr, fpr, step, walltime):
+    """Package raw data to one roc_curve.
+    Args:
+        tag (string): Data identifier
+        tp (list): True Positive.
+        fp (list): False Positive.
+        tn (list): True Negative.
+        fn (list): False Negative.
+        tpr (list): true positive rate:
+        fpr (list): false positive rate.
+        step (int): Step of roc_curve
+        walltime (int): Wall time of roc_curve
+        num_thresholds (int): Number of thresholds used to draw the curve.
+        weights (float): Multiple of data to display on the curve.
+    Return:
+        Package with format of record_pb2.Record
+    """
+
+    """
+    if isinstance(tp, np.ndarray):
+        tp = tp.astype(int).tolist()
+    if isinstance(fp, np.ndarray):
+        fp = fp.astype(int).tolist()
+    if isinstance(tn, np.ndarray):
+        tn = tn.astype(int).tolist()
+    if isinstance(fn, np.ndarray):
+        fn = fn.astype(int).tolist()
+    if isinstance(tpr, np.ndarray):
+        tpr = tpr.astype(int).tolist()
+    if isinstance(fpr, np.ndarray):
+        fpr = fpr.astype(int).tolist()
+    """
+    roc_curve = Record.ROC_Curve(TP=tp,
+                             FP=fp,
+                             TN=tn,
+                             FN=fn,
+                             tpr=tpr,
+                             fpr=fpr)
+    return Record(values=[
+        Record.Value(
+            id=step, tag=tag, timestamp=walltime, roc_curve=roc_curve)
+    ])
+    

visualdl/proto/record.proto

@@ -43,6 +43,15 @@ message Record {
     repeated double precision = 5;
     repeated double recall = 6;
   }
+  
+  message ROC_Curve {
+    repeated int64 TP = 1 [packed = true];
+    repeated int64 FP = 2 [packed = true];
+    repeated int64 TN = 3 [packed = true];
+    repeated int64 FN = 4 [packed = true];
+    repeated double tpr = 5;
+    repeated double fpr = 6;
+  }
 
   message MetaData {
     string display_name = 1;
@@ -60,6 +69,7 @@ message Record {
       Histogram histogram = 8;
       PRCurve pr_curve = 9;
       MetaData meta_data = 10;
+      ROC_Curve roc_curve = 11;
     }
   }
 

visualdl/proto/record_pb2.py

 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: record.proto
-
+"""Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
 from google.protobuf import reflection as _reflection
@@ -18,7 +18,8 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   package='visualdl',
   syntax='proto3',
   serialized_options=None,
-  serialized_pb=b'\n\x0crecord.proto\x12\x08visualdl\"\xc8\x07\n\x06Record\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.visualdl.Record.Value\x1a%\n\x05Image\x12\x1c\n\x14\x65ncoded_image_string\x18\x04 \x01(\x0c\x1a}\n\x05\x41udio\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x14\n\x0cnum_channels\x18\x02 \x01(\x03\x12\x15\n\rlength_frames\x18\x03 \x01(\x03\x12\x1c\n\x14\x65ncoded_audio_string\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\x1a+\n\tEmbedding\x12\r\n\x05label\x18\x01 \x03(\t\x12\x0f\n\x07vectors\x18\x02 \x03(\x02\x1aP\n\nEmbeddings\x12.\n\nembeddings\x18\x01 \x03(\x0b\x32\x1a.visualdl.Record.Embedding\x12\x12\n\nlabel_meta\x18\x02 \x03(\t\x1a\x43\n\x10\x62ytes_embeddings\x12\x16\n\x0e\x65ncoded_labels\x18\x01 \x01(\x0c\x12\x17\n\x0f\x65ncoded_vectors\x18\x02 \x01(\x0c\x1a\x34\n\tHistogram\x12\x10\n\x04hist\x18\x01 \x03(\x01\x42\x02\x10\x01\x12\x15\n\tbin_edges\x18\x02 \x03(\x01\x42\x02\x10\x01\x1al\n\x07PRCurve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x11\n\tprecision\x18\x05 \x03(\x01\x12\x0e\n\x06recall\x18\x06 \x03(\x01\x1a \n\x08MetaData\x12\x14\n\x0c\x64isplay_name\x18\x01 \x01(\t\x1a\xe5\x02\n\x05Value\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12\x0f\n\x05value\x18\x04 \x01(\x02H\x00\x12\'\n\x05image\x18\x05 \x01(\x0b\x32\x16.visualdl.Record.ImageH\x00\x12\'\n\x05\x61udio\x18\x06 \x01(\x0b\x32\x16.visualdl.Record.AudioH\x00\x12\x31\n\nembeddings\x18\x07 \x01(\x0b\x32\x1b.visualdl.Record.EmbeddingsH\x00\x12/\n\thistogram\x18\x08 \x01(\x0b\x32\x1a.visualdl.Record.HistogramH\x00\x12,\n\x08pr_curve\x18\t \x01(\x0b\x32\x18.visualdl.Record.PRCurveH\x00\x12.\n\tmeta_data\x18\n \x01(\x0b\x32\x19.visualdl.Record.MetaDataH\x00\x42\x0b\n\tone_valueb\x06proto3'
+  create_key=_descriptor._internal_create_key,
+  serialized_pb=b'\n\x0crecord.proto\x12\x08visualdl\"\xe0\x08\n\x06Record\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.visualdl.Record.Value\x1a%\n\x05Image\x12\x1c\n\x14\x65ncoded_image_string\x18\x04 \x01(\x0c\x1a}\n\x05\x41udio\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x14\n\x0cnum_channels\x18\x02 \x01(\x03\x12\x15\n\rlength_frames\x18\x03 \x01(\x03\x12\x1c\n\x14\x65ncoded_audio_string\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\x1a+\n\tEmbedding\x12\r\n\x05label\x18\x01 \x03(\t\x12\x0f\n\x07vectors\x18\x02 \x03(\x02\x1aP\n\nEmbeddings\x12.\n\nembeddings\x18\x01 \x03(\x0b\x32\x1a.visualdl.Record.Embedding\x12\x12\n\nlabel_meta\x18\x02 \x03(\t\x1a\x43\n\x10\x62ytes_embeddings\x12\x16\n\x0e\x65ncoded_labels\x18\x01 \x01(\x0c\x12\x17\n\x0f\x65ncoded_vectors\x18\x02 \x01(\x0c\x1a\x34\n\tHistogram\x12\x10\n\x04hist\x18\x01 \x03(\x01\x42\x02\x10\x01\x12\x15\n\tbin_edges\x18\x02 \x03(\x01\x42\x02\x10\x01\x1al\n\x07PRCurve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x11\n\tprecision\x18\x05 \x03(\x01\x12\x0e\n\x06recall\x18\x06 \x03(\x01\x1a\x65\n\tROC_Curve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x0b\n\x03tpr\x18\x05 \x03(\x01\x12\x0b\n\x03\x66pr\x18\x06 \x03(\x01\x1a \n\x08MetaData\x12\x14\n\x0c\x64isplay_name\x18\x01 \x01(\t\x1a\x96\x03\n\x05Value\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12\x0f\n\x05value\x18\x04 \x01(\x02H\x00\x12\'\n\x05image\x18\x05 \x01(\x0b\x32\x16.visualdl.Record.ImageH\x00\x12\'\n\x05\x61udio\x18\x06 \x01(\x0b\x32\x16.visualdl.Record.AudioH\x00\x12\x31\n\nembeddings\x18\x07 \x01(\x0b\x32\x1b.visualdl.Record.EmbeddingsH\x00\x12/\n\thistogram\x18\x08 \x01(\x0b\x32\x1a.visualdl.Record.HistogramH\x00\x12,\n\x08pr_curve\x18\t \x01(\x0b\x32\x18.visualdl.Record.PRCurveH\x00\x12.\n\tmeta_data\x18\n \x01(\x0b\x32\x19.visualdl.Record.MetaDataH\x00\x12/\n\troc_curve\x18\x0b \x01(\x0b\x32\x1a.visualdl.Record.ROC_CurveH\x00\x42\x0b\n\tone_valueb\x06proto3'
 )
 
 
@@ -30,6 +31,7 @@ _RECORD_IMAGE = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='encoded_image_string', full_name='visualdl.Record.Image.encoded_image_string', index=0,
@@ -37,7 +39,7 @@ _RECORD_IMAGE = _descriptor.Descriptor(
       has_default_value=False, default_value=b"",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -60,6 +62,7 @@ _RECORD_AUDIO = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='sample_rate', full_name='visualdl.Record.Audio.sample_rate', index=0,
@@ -67,35 +70,35 @@ _RECORD_AUDIO = _descriptor.Descriptor(
       has_default_value=False, default_value=float(0),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='num_channels', full_name='visualdl.Record.Audio.num_channels', index=1,
       number=2, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='length_frames', full_name='visualdl.Record.Audio.length_frames', index=2,
       number=3, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='encoded_audio_string', full_name='visualdl.Record.Audio.encoded_audio_string', index=3,
       number=4, type=12, cpp_type=9, label=1,
       has_default_value=False, default_value=b"",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='content_type', full_name='visualdl.Record.Audio.content_type', index=4,
       number=5, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -118,6 +121,7 @@ _RECORD_EMBEDDING = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='label', full_name='visualdl.Record.Embedding.label', index=0,
@@ -125,14 +129,14 @@ _RECORD_EMBEDDING = _descriptor.Descriptor(
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='vectors', full_name='visualdl.Record.Embedding.vectors', index=1,
       number=2, type=2, cpp_type=6, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -155,6 +159,7 @@ _RECORD_EMBEDDINGS = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='embeddings', full_name='visualdl.Record.Embeddings.embeddings', index=0,
@@ -162,14 +167,14 @@ _RECORD_EMBEDDINGS = _descriptor.Descriptor(
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='label_meta', full_name='visualdl.Record.Embeddings.label_meta', index=1,
       number=2, type=9, cpp_type=9, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -192,6 +197,7 @@ _RECORD_BYTES_EMBEDDINGS = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='encoded_labels', full_name='visualdl.Record.bytes_embeddings.encoded_labels', index=0,
@@ -199,14 +205,14 @@ _RECORD_BYTES_EMBEDDINGS = _descriptor.Descriptor(
       has_default_value=False, default_value=b"",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='encoded_vectors', full_name='visualdl.Record.bytes_embeddings.encoded_vectors', index=1,
       number=2, type=12, cpp_type=9, label=1,
       has_default_value=False, default_value=b"",
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -229,6 +235,7 @@ _RECORD_HISTOGRAM = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='hist', full_name='visualdl.Record.Histogram.hist', index=0,
@@ -236,14 +243,14 @@ _RECORD_HISTOGRAM = _descriptor.Descriptor(
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=b'\020\001', file=DESCRIPTOR),
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='bin_edges', full_name='visualdl.Record.Histogram.bin_edges', index=1,
       number=2, type=1, cpp_type=5, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=b'\020\001', file=DESCRIPTOR),
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -266,6 +273,7 @@ _RECORD_PRCURVE = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='TP', full_name='visualdl.Record.PRCurve.TP', index=0,
@@ -273,42 +281,42 @@ _RECORD_PRCURVE = _descriptor.Descriptor(
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=b'\020\001', file=DESCRIPTOR),
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='FP', full_name='visualdl.Record.PRCurve.FP', index=1,
       number=2, type=3, cpp_type=2, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=b'\020\001', file=DESCRIPTOR),
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='TN', full_name='visualdl.Record.PRCurve.TN', index=2,
       number=3, type=3, cpp_type=2, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=b'\020\001', file=DESCRIPTOR),
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='FN', full_name='visualdl.Record.PRCurve.FN', index=3,
       number=4, type=3, cpp_type=2, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=b'\020\001', file=DESCRIPTOR),
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='precision', full_name='visualdl.Record.PRCurve.precision', index=4,
       number=5, type=1, cpp_type=5, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='recall', full_name='visualdl.Record.PRCurve.recall', index=5,
       number=6, type=1, cpp_type=5, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -325,12 +333,79 @@ _RECORD_PRCURVE = _descriptor.Descriptor(
   serialized_end=601,
 )
 
+_RECORD_ROC_CURVE = _descriptor.Descriptor(
+  name='ROC_Curve',
+  full_name='visualdl.Record.ROC_Curve',
+  filename=None,
+  file=DESCRIPTOR,
+  containing_type=None,
+  create_key=_descriptor._internal_create_key,
+  fields=[
+    _descriptor.FieldDescriptor(
+      name='TP', full_name='visualdl.Record.ROC_Curve.TP', index=0,
+      number=1, type=3, cpp_type=2, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='FP', full_name='visualdl.Record.ROC_Curve.FP', index=1,
+      number=2, type=3, cpp_type=2, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='TN', full_name='visualdl.Record.ROC_Curve.TN', index=2,
+      number=3, type=3, cpp_type=2, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='FN', full_name='visualdl.Record.ROC_Curve.FN', index=3,
+      number=4, type=3, cpp_type=2, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=b'\020\001', file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='tpr', full_name='visualdl.Record.ROC_Curve.tpr', index=4,
+      number=5, type=1, cpp_type=5, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='fpr', full_name='visualdl.Record.ROC_Curve.fpr', index=5,
+      number=6, type=1, cpp_type=5, label=3,
+      has_default_value=False, default_value=[],
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+  ],
+  extensions=[
+  ],
+  nested_types=[],
+  enum_types=[
+  ],
+  serialized_options=None,
+  is_extendable=False,
+  syntax='proto3',
+  extension_ranges=[],
+  oneofs=[
+  ],
+  serialized_start=603,
+  serialized_end=704,
+)
+
 _RECORD_METADATA = _descriptor.Descriptor(
   name='MetaData',
   full_name='visualdl.Record.MetaData',
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='display_name', full_name='visualdl.Record.MetaData.display_name', index=0,
@@ -338,7 +413,7 @@ _RECORD_METADATA = _descriptor.Descriptor(
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -351,8 +426,8 @@ _RECORD_METADATA = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=603,
-  serialized_end=635,
+  serialized_start=706,
+  serialized_end=738,
 )
 
 _RECORD_VALUE = _descriptor.Descriptor(
@@ -361,6 +436,7 @@ _RECORD_VALUE = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='id', full_name='visualdl.Record.Value.id', index=0,
@@ -368,70 +444,77 @@ _RECORD_VALUE = _descriptor.Descriptor(
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='tag', full_name='visualdl.Record.Value.tag', index=1,
       number=2, type=9, cpp_type=9, label=1,
       has_default_value=False, default_value=b"".decode('utf-8'),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='timestamp', full_name='visualdl.Record.Value.timestamp', index=2,
       number=3, type=3, cpp_type=2, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='value', full_name='visualdl.Record.Value.value', index=3,
       number=4, type=2, cpp_type=6, label=1,
       has_default_value=False, default_value=float(0),
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='image', full_name='visualdl.Record.Value.image', index=4,
       number=5, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='audio', full_name='visualdl.Record.Value.audio', index=5,
       number=6, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='embeddings', full_name='visualdl.Record.Value.embeddings', index=6,
       number=7, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='histogram', full_name='visualdl.Record.Value.histogram', index=7,
       number=8, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='pr_curve', full_name='visualdl.Record.Value.pr_curve', index=8,
       number=9, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
     _descriptor.FieldDescriptor(
       name='meta_data', full_name='visualdl.Record.Value.meta_data', index=9,
       number=10, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
+    _descriptor.FieldDescriptor(
+      name='roc_curve', full_name='visualdl.Record.Value.roc_curve', index=10,
+      number=11, type=11, cpp_type=10, label=1,
+      has_default_value=False, default_value=None,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
@@ -445,10 +528,12 @@ _RECORD_VALUE = _descriptor.Descriptor(
   oneofs=[
     _descriptor.OneofDescriptor(
       name='one_value', full_name='visualdl.Record.Value.one_value',
-      index=0, containing_type=None, fields=[]),
+      index=0, containing_type=None,
+      create_key=_descriptor._internal_create_key,
+    fields=[]),
   ],
-  serialized_start=638,
-  serialized_end=995,
+  serialized_start=741,
+  serialized_end=1147,
 )
 
 _RECORD = _descriptor.Descriptor(
@@ -457,6 +542,7 @@ _RECORD = _descriptor.Descriptor(
   filename=None,
   file=DESCRIPTOR,
   containing_type=None,
+  create_key=_descriptor._internal_create_key,
   fields=[
     _descriptor.FieldDescriptor(
       name='values', full_name='visualdl.Record.values', index=0,
@@ -464,11 +550,11 @@ _RECORD = _descriptor.Descriptor(
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
-      serialized_options=None, file=DESCRIPTOR),
+      serialized_options=None, file=DESCRIPTOR,  create_key=_descriptor._internal_create_key),
   ],
   extensions=[
   ],
-  nested_types=[_RECORD_IMAGE, _RECORD_AUDIO, _RECORD_EMBEDDING, _RECORD_EMBEDDINGS, _RECORD_BYTES_EMBEDDINGS, _RECORD_HISTOGRAM, _RECORD_PRCURVE, _RECORD_METADATA, _RECORD_VALUE, ],
+  nested_types=[_RECORD_IMAGE, _RECORD_AUDIO, _RECORD_EMBEDDING, _RECORD_EMBEDDINGS, _RECORD_BYTES_EMBEDDINGS, _RECORD_HISTOGRAM, _RECORD_PRCURVE, _RECORD_ROC_CURVE, _RECORD_METADATA, _RECORD_VALUE, ],
   enum_types=[
   ],
   serialized_options=None,
@@ -478,7 +564,7 @@ _RECORD = _descriptor.Descriptor(
   oneofs=[
   ],
   serialized_start=27,
-  serialized_end=995,
+  serialized_end=1147,
 )
 
 _RECORD_IMAGE.containing_type = _RECORD
@@ -489,6 +575,7 @@ _RECORD_EMBEDDINGS.containing_type = _RECORD
 _RECORD_BYTES_EMBEDDINGS.containing_type = _RECORD
 _RECORD_HISTOGRAM.containing_type = _RECORD
 _RECORD_PRCURVE.containing_type = _RECORD
+_RECORD_ROC_CURVE.containing_type = _RECORD
 _RECORD_METADATA.containing_type = _RECORD
 _RECORD_VALUE.fields_by_name['image'].message_type = _RECORD_IMAGE
 _RECORD_VALUE.fields_by_name['audio'].message_type = _RECORD_AUDIO
@@ -496,6 +583,7 @@ _RECORD_VALUE.fields_by_name['embeddings'].message_type = _RECORD_EMBEDDINGS
 _RECORD_VALUE.fields_by_name['histogram'].message_type = _RECORD_HISTOGRAM
 _RECORD_VALUE.fields_by_name['pr_curve'].message_type = _RECORD_PRCURVE
 _RECORD_VALUE.fields_by_name['meta_data'].message_type = _RECORD_METADATA
+_RECORD_VALUE.fields_by_name['roc_curve'].message_type = _RECORD_ROC_CURVE
 _RECORD_VALUE.containing_type = _RECORD
 _RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
   _RECORD_VALUE.fields_by_name['value'])
@@ -518,6 +606,9 @@ _RECORD_VALUE.fields_by_name['pr_curve'].containing_oneof = _RECORD_VALUE.oneofs
 _RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
   _RECORD_VALUE.fields_by_name['meta_data'])
 _RECORD_VALUE.fields_by_name['meta_data'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
+_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
+  _RECORD_VALUE.fields_by_name['roc_curve'])
+_RECORD_VALUE.fields_by_name['roc_curve'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
 _RECORD.fields_by_name['values'].message_type = _RECORD_VALUE
 DESCRIPTOR.message_types_by_name['Record'] = _RECORD
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
@@ -573,6 +664,13 @@ Record = _reflection.GeneratedProtocolMessageType('Record', (_message.Message,),
     })
   ,
 
+  'ROC_Curve' : _reflection.GeneratedProtocolMessageType('ROC_Curve', (_message.Message,), {
+    'DESCRIPTOR' : _RECORD_ROC_CURVE,
+    '__module__' : 'record_pb2'
+    # @@protoc_insertion_point(class_scope:visualdl.Record.ROC_Curve)
+    })
+  ,
+
   'MetaData' : _reflection.GeneratedProtocolMessageType('MetaData', (_message.Message,), {
     'DESCRIPTOR' : _RECORD_METADATA,
     '__module__' : 'record_pb2'
@@ -598,6 +696,7 @@ _sym_db.RegisterMessage(Record.Embeddings)
 _sym_db.RegisterMessage(Record.bytes_embeddings)
 _sym_db.RegisterMessage(Record.Histogram)
 _sym_db.RegisterMessage(Record.PRCurve)
+_sym_db.RegisterMessage(Record.ROC_Curve)
 _sym_db.RegisterMessage(Record.MetaData)
 _sym_db.RegisterMessage(Record.Value)
 
@@ -608,4 +707,8 @@ _RECORD_PRCURVE.fields_by_name['TP']._options = None
 _RECORD_PRCURVE.fields_by_name['FP']._options = None
 _RECORD_PRCURVE.fields_by_name['TN']._options = None
 _RECORD_PRCURVE.fields_by_name['FN']._options = None
+_RECORD_ROC_CURVE.fields_by_name['TP']._options = None
+_RECORD_ROC_CURVE.fields_by_name['FP']._options = None
+_RECORD_ROC_CURVE.fields_by_name['TN']._options = None
+_RECORD_ROC_CURVE.fields_by_name['FN']._options = None
 # @@protoc_insertion_point(module_scope)

visualdl/reader/reader.py

@@ -171,6 +171,8 @@ class LogReader(object):
                 component = "histogram"
             elif "pr_curve" == value_type:
                 component = "pr_curve"
+            elif "roc_curve" == value_type:
+                component = "roc_curve"
             elif "meta_data" == value_type:
                 self.update_meta_data(record)
                 component = "meta_data"

visualdl/server/api.py

@@ -112,6 +112,9 @@ class Api(object):
     @result()
     def pr_curve_tags(self):
         return self._get_with_retry('data/plugin/pr_curves/tags', lib.get_pr_curve_tags)
+    @result()
+    def roc_curve_tags(self):
+        return self._get_with_retry('data/plugin/roc_curves/tags', lib.get_roc_curve_tags)
 
     @result()
     def scalar_list(self, run, tag):
@@ -178,11 +181,18 @@ class Api(object):
     def pr_curves_pr_curve(self, run, tag):
         key = os.path.join('data/plugin/pr_curves/pr_curve', run, tag)
         return self._get_with_retry(key, lib.get_pr_curve, run, tag)
-
+    @result()
+    def roc_curves_roc_curve(self, run, tag):
+        key = os.path.join('data/plugin/roc_curves/roc_curve', run, tag)
+        return self._get_with_retry(key, lib.get_roc_curve, run, tag)
     @result()
     def pr_curves_steps(self, run):
         key = os.path.join('data/plugin/pr_curves/steps', run)
         return self._get_with_retry(key, lib.get_pr_curve_step, run)
+    @result()
+    def roc_curves_steps(self, run):
+        key = os.path.join('data/plugin/roc_curves/steps', run)
+        return self._get_with_retry(key, lib.get_roc_curve_step, run)
 
     @result('application/octet-stream', lambda s: {"Content-Disposition": 'attachment; filename="%s"' % s.model_name} if len(s.model_name) else None)
     def graph_graph(self):
@@ -203,6 +213,7 @@ def create_api_call(logdir, model, cache_timeout):
         'embedding/tags': (api.embedding_tags, []),
         'histogram/tags': (api.histogram_tags, []),
         'pr-curve/tags': (api.pr_curve_tags, []),
+        'roc-curve/tags': (api.roc_curve_tags, []),
         'scalar/list': (api.scalar_list, ['run', 'tag']),
         'scalar/data': (api.scalar_data, ['run', 'tag']),
         'image/list': (api.image_list, ['run', 'tag']),
@@ -216,7 +227,9 @@ def create_api_call(logdir, model, cache_timeout):
         'histogram/list': (api.histogram_list, ['run', 'tag']),
         'graph/graph': (api.graph_graph, []),
         'pr-curve/list': (api.pr_curves_pr_curve, ['run', 'tag']),
-        'pr-curve/steps': (api.pr_curves_steps, ['run'])
+        'roc-curve/list': (api.roc_curves_roc_curve, ['run', 'tag']),
+        'pr-curve/steps': (api.pr_curves_steps, ['run']),
+        'roc-curve/steps': (api.roc_curves_steps, ['run'])
     }
 
     def call(path: str, args):

visualdl/server/data_manager.py

@@ -24,6 +24,7 @@ DEFAULT_PLUGIN_MAXSIZE = {
     "embeddings": 50000000,
     "audio": 10,
     "pr_curve": 300,
+    "roc_curve": 300,
     "meta_data": 100
 }
 
@@ -346,6 +347,8 @@ class DataManager(object):
             Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["audio"]),
             "pr_curve":
             Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["pr_curve"]),
+            "roc_curve":
+            Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["roc_curve"]),
             "meta_data":
             Reservoir(max_size=DEFAULT_PLUGIN_MAXSIZE["meta_data"])
         }

visualdl/server/lib.py

@@ -198,6 +198,28 @@ def get_pr_curve(log_reader, run, tag):
                         list(pr_curve.FN),
                         num_thresholds])
     return results
+ 
+    
+def get_roc_curve(log_reader, run, tag):
+    run = log_reader.name2tags[run] if run in log_reader.name2tags else run
+    log_reader.load_new_data()
+    records = log_reader.data_manager.get_reservoir("roc_curve").get_items(
+        run, decode_tag(tag))
+    results = []
+    for item in records:
+        roc_curve = item.roc_curve
+        length = len(roc_curve.tpr)
+        num_thresholds = [float(v) / length for v in range(1, length + 1)]
+        results.append([s2ms(item.timestamp),
+                        item.id,
+                        list(roc_curve.tpr),
+                        list(roc_curve.fpr),
+                        list(roc_curve.TP),
+                        list(roc_curve.FP),
+                        list(roc_curve.TN),
+                        list(roc_curve.FN),
+                        num_thresholds])
+    return results
 
 
 def get_pr_curve_step(log_reader, run, tag=None):
@@ -212,6 +234,18 @@ def get_pr_curve_step(log_reader, run, tag=None):
     return results
 
 
+def get_roc_curve_step(log_reader, run, tag=None):
+    fake_run = run
+    run = log_reader.name2tags[run] if run in log_reader.name2tags else run
+    run2tag = get_roc_curve_tags(log_reader)
+    tag = run2tag['tags'][run2tag['runs'].index(fake_run)][0]
+    log_reader.load_new_data()
+    records = log_reader.data_manager.get_reservoir("roc_curve").get_items(
+        run, decode_tag(tag))
+    results = [[s2ms(item.timestamp), item.id] for item in records]
+    return results
+
+  
 def get_embeddings_list(log_reader):
     run2tag = get_logs(log_reader, 'embeddings')
 
@@ -266,30 +300,6 @@ def get_embedding_tensors(log_reader, name):
     return vectors
 
 
-def get_embeddings(log_reader, run, tag, reduction, dimension=2):
-    run = log_reader.name2tags[run] if run in log_reader.name2tags else run
-    log_reader.load_new_data()
-    records = log_reader.data_manager.get_reservoir("embeddings").get_items(
-        run, decode_tag(tag))
-
-    labels = []
-    vectors = []
-    for item in records[0].embeddings.embeddings:
-        labels.append(item.label)
-        vectors.append(item.vectors)
-    vectors = np.array(vectors)
-
-    if reduction == 'tsne':
-        import visualdl.server.tsne as tsne
-        low_dim_embs = tsne.tsne(
-            vectors, dimension, initial_dims=50, perplexity=30.0)
-
-    elif reduction == 'pca':
-        low_dim_embs = simple_pca(vectors, dimension)
-
-    return {"embedding": low_dim_embs.tolist(), "labels": labels}
-
-
 def get_histogram(log_reader, run, tag):
     run = log_reader.name2tags[run] if run in log_reader.name2tags else run
     log_reader.load_new_data()

visualdl/writer/writer.py

@@ -16,7 +16,7 @@ import os
 import time
 import numpy as np
 from visualdl.writer.record_writer import RecordFileWriter
-from visualdl.component.base_component import scalar, image, embedding, audio, histogram, pr_curve, meta_data
+from visualdl.component.base_component import scalar, image, embedding, audio, histogram, pr_curve, roc_curve, meta_data
 
 
 class DummyFileWriter(object):
@@ -380,6 +380,48 @@ class LogWriter(object):
                 num_thresholds=num_thresholds,
                 weights=weights
                 ))
+    def add_roc_curve(self,
+                     tag,
+                     labels,
+                     predictions,
+                     step,
+                     num_thresholds=10,
+                     weights=None,
+                     walltime=None):
+        """Add an ROC curve to vdl record file.
+        Args:
+            tag (string): Data identifier
+            labels (numpy.ndarray or list): Binary labels for each element.
+            predictions (numpy.ndarray or list): The probability that an element
+                be classified as true.
+            step (int): Step of pr curve.
+            weights (float): Multiple of data to display on the curve.
+            num_thresholds (int): Number of thresholds used to draw the curve.
+            walltime (int): Wall time of pr curve.
+        Example:
+            with LogWriter(logdir="./log/roc_curve_test/train") as writer:
+                for index in range(3):
+                    labels = np.random.randint(2, size=100)
+                    predictions = np.random.rand(100)
+                    writer.add_roc_curve(tag='default',
+                                        labels=labels,
+                                        predictions=predictions,
+                                        step=index)
+        """
+        if '%' in tag:
+            raise RuntimeError("% can't appear in tag!")
+        walltime = round(time.time() * 1000) if walltime is None else walltime
+        self._get_file_writer().add_record(
+            roc_curve(
+                tag=tag,
+                labels=labels,
+                predictions=predictions,
+                step=step,
+                walltime=walltime,
+                num_thresholds=num_thresholds,
+                weights=weights
+                ))
+
 
     def flush(self):
         """Flush all data in cache to disk.