Add Audio UI in app and audio endpoint, test by adding mock audio tags (#352)

frontend/mock/data/plugin/audio/audio.js

+/**
+ * get mock data
+ *
+ * @param {string} path request path
+ * @param {Object} queryParam query params
+ * @param {Object} postParam post params
+ * @return {Object}
+ */
+module.exports = function (path, queryParam, postParam) {
+    return {
+        // moock delay
+        _timeout: 0,
+        // mock http status
+        _status: 200,
+        // mock response data
+        _data: {
+            status: 0,
+            msg: 'SUCCESS',
+            data: [
+                {
+                    "wall_time": 1512549785.061623,
+                    "step": 60,
+                    "query": "sample=0&index=0&tag=input_reshape%2Finput%2Faudio%2F0&run=test",
+                    "width": 28,
+                    "height": 28
+                },
+                {"wall_time": 1512886109.672786, "step": 60, "query": "sample=0&index=1&tag=input_reshape%2Finput%2Faudio%2F0&run=test"},
+                {"wall_time": 1512886124.266915, "step": 210, "query": "sample=0&index=2&tag=input_reshape%2Finput%2Faudio%2F0&run=test"},
+                {"wall_time": 1512886138.898628, "step": 330, "query": "sample=0&index=3&tag=input_reshape%2Finput%2Faudio%2F0&run=test"},
+                {"wall_time": 1512886139.883663, "step": 340, "query": "sample=0&index=4&tag=input_reshape%2Finput%2Faudio%2F0&run=test"},
+                {"wall_time": 1512886147.195567, "step": 410, "query": "sample=0&index=5&tag=input_reshape%2Finput%2Faudio%2F0&run=test"},
+                {"wall_time": 1512886156.47856, "step": 500, "query": "sample=0&index=6&tag=input_reshape%2Finput%2Faudio%2F0&run=test"},
+                {"wall_time": 1512886187.82793, "step": 810, "query": "sample=0&index=7&tag=input_reshape%2Finput%2Faudio%2F0&run=test"},
+                {"wall_time": 1512886200.386198, "step": 950, "query": "sample=0&index=8&tag=input_reshape%2Finput%2Faudio%2F0&run=test"},
+                {"wall_time": 1512886204.224405, "step": 990, "query": "sample=0&index=9&tag=input_reshape%2Finput%2Faudio%2F0&run=test"}
+            ]
+        }
+    };
+};

frontend/mock/data/plugin/audio/individualaudio.js

+/**
+ * get mock data
+ *
+ * @param {string} path request path
+ * @param {Object} queryParam query params
+ * @param {Object} postParam post params
+ * @return {Object}
+ */
+module.exports = function (path, queryParam, postParam) {
+    return {
+        // moock delay
+        _timeout: 0,
+        // mock http status
+        _status: 200,
+        // mock response data
+        _data: {
+            status: 0,
+            msg: 'SUCCESS',
+            data: ''
+        }
+    };
+};

frontend/mock/data/plugin/audio/tags.js

+/**
+ * get mock data
+ *
+ * @param {string} path request path
+ * @param {Object} queryParam query params
+ * @param {Object} postParam post params
+ * @return {Object}
+ */
+ module.exports = function (path, queryParam, postParam) {
+    return {
+        // moock delay
+        _timeout: 0,
+        // mock http status
+        _status: 200,
+        // mock response data
+        _data: {
+            status: 0,
+            msg: 'SUCCESS',
+            data: {
+                "test": {
+                    "input_reshape/input/audio/7": {
+                        "displayName": "input_reshape/input/audio/7",
+                        "description": "", "samples": 1
+                    },
+                    "input_reshape/input/audio/4": {
+                        "displayName": "input_reshape/input/audio/4",
+                        "description": "",
+                        "samples": 1
+                    },
+                    "input_reshape/input/audio/5": {
+                        "displayName": "input_reshape/input/audio/5",
+                        "description": "", "samples": 1
+                    }, "input_reshape/input/audio/2": {"displayName": "input_reshape/input/audio/2", "description": "", "samples": 1}, "input_reshape/input/audio/3": {"displayName": "input_reshape/input/audio/3", "description": "", "samples": 1}, "input_reshape/input/audio/0": {"displayName": "input_reshape/input/audio/0", "description": "", "samples": 1}, "input_reshape/input/audio/1": {"displayName": "input_reshape/input/audio/1", "description": "", "samples": 1}, "input_reshape/input/audio/8": {"displayName": "input_reshape/input/audio/8", "description": "", "samples": 1}, "input_reshape/input/audio/9": {"displayName": "input_reshape/input/audio/9", "description": "", "samples": 1}}, "train": {"input_reshape/input/audio/6": {"displayName": "input_reshape/input/audio/6", "description": "", "samples": 1}, "input_reshape/input/audio/7": {"displayName": "input_reshape/input/audio/7", "description": "", "samples": 1}, "input_reshape/input/audio/4": {"displayName": "input_reshape/input/audio/4", "description": "", "samples": 1}, "input_reshape/input/audio/5": {"displayName": "input_reshape/input/audio/5", "description": "", "samples": 1}, "input_reshape/input/audio/2": {"displayName": "input_reshape/input/audio/2", "description": "", "samples": 1}, "input_reshape/input/audio/3": {"displayName": "input_reshape/input/audio/3", "description": "", "samples": 1}, "input_reshape/input/audio/0": {"displayName": "input_reshape/input/audio/0", "description": "", "samples": 1}, "input_reshape/input/audio/1": {"displayName": "input_reshape/input/audio/1", "description": "", "samples": 1}, "input_reshape/input/audio/8": {"displayName": "input_reshape/input/audio/8", "description": "", "samples": 1}, "input_reshape/input/audio/9": {"displayName": "input_reshape/input/audio/9", "description": "", "samples": 1}}}
+        }
+    };
+};

frontend/src/audio/Audio.vue

+<template>
+    <div class="visual-dl-page-container">
+        <div class="visual-dl-page-left">
+            <ui-audio-container
+                :expand="true"
+                :config="filteredConfig"
+                :runsItems="runsItems"
+                :tagList="filteredTagsList"
+                :title="'Tags matching ' + config.groupNameReg"
+            ></ui-audio-container>
+            <ui-audio-container
+                v-for="item in groupedTags"
+                :key="item.group"
+                :config="filteredConfig"
+                :runsItems="runsItems"
+                :tagList="item.tags"
+                :title="item.group"
+            ></ui-audio-container>
+        </div>
+        <div class="visual-dl-page-right">
+            <div class="visual-dl-page-config-container">
+                <ui-config :runsItems="runsItems" :config="config"
+                ></ui-config>
+            </div>
+        </div>
+    </div>
+</template>
+
+<script>
+
+import {getPluginAudioTags, getRuns} from '../service';
+import {debounce, flatten, uniq, isArray} from 'lodash';
+import autoAdjustHeight from '../common/util/autoAdjustHeight';
+
+import Config from './ui/Config'
+import AudioPanelContainer from './ui/AudioPanelContainer';
+
+export default {
+    name: 'Images',
+    components: {
+        'ui-config': Config,
+        'ui-audio-container': AudioPanelContainer
+    },
+    data () {
+        return {
+            runsArray: [],
+            tags: [],
+            config: {
+                groupNameReg: '.*',
+                isActualImageSize: false,
+                runs: [],
+                running: true
+            },
+            filteredTagsList: []
+        }
+    },
+    computed: {
+        runsItems() {
+            let runsArray = this.runsArray || [];
+            return runsArray.map(item => {
+                return {
+                    name: item,
+                    value: item
+                };
+            });
+        },
+        tagsList() {
+            let tags = this.tags;
+
+            let runs = Object.keys(tags);
+            let tagsArray = runs.map(run => Object.keys(tags[run]));
+            let allUniqTags = uniq(flatten(tagsArray));
+
+            // get the data for every chart
+            return allUniqTags.map(tag => {
+                let tagList = runs.map(run => {
+                    return {
+                        run,
+                        tag: tags[run][tag]
+                    };
+                }).filter(item => item.tag !== undefined);
+                return {
+                    tagList,
+                    tag,
+                    group: tag.split('/')[0]
+                };
+            });
+        },
+        groupedTags() {
+            let tagsList = this.tagsList || [];
+            // put data in group
+            let groupData = {};
+            tagsList.forEach(item => {
+                let group = item.group;
+                if (groupData[group] === undefined) {
+                    groupData[group] = [];
+                    groupData[group].push(item);
+                }
+                else {
+                    groupData[group].push(item);
+                }
+            });
+
+            // to array
+            let groups = Object.keys(groupData);
+            return groups.map(group => {
+                return {
+                    group,
+                    tags: groupData[group]
+                };
+            });
+        },
+        filteredConfig() {
+            let config = this.config || {};
+            let filteredConfig = {};
+            Object.keys(config).forEach(key => {
+                let val = config[key];
+                filteredConfig[key] = val;
+            });
+            return filteredConfig;
+        }
+    },
+    created() {
+        getPluginAudioTags().then(({errno, data}) => {
+            this.tags = data;
+
+            // filter when inited
+            let groupNameReg = this.config.groupNameReg;
+            this.filterTagsList(groupNameReg);
+        });
+
+        getRuns().then(({errno, data}) => {
+            this.runsArray = data;
+            this.config.runs = data;
+        });
+    },
+    mounted() {
+	    autoAdjustHeight();
+    },
+    watch: {
+        'config.groupNameReg': function(val) {
+            this.throttledFilterTagsList()
+        }
+    },
+    methods:{
+        filterTagsList(groupNameReg) {
+            if (!groupNameReg) {
+                this.filteredTagsList = [];
+                return;
+            }
+            let tagsList = this.tagsList || [];
+            let regExp = new RegExp(groupNameReg);
+            this.filteredTagsList = tagsList.filter(item => regExp.test(item.tag));
+        },
+        throttledFilterTagsList: _.debounce(
+            function() {
+                this.filterTagsList(this.config.groupNameReg)
+            }, 300
+        ),
+    }
+};
+
+</script>
+
+<style lang="stylus">
+
+</style>

frontend/src/audio/index.js

+import {router} from 'san-router';
+
+import Audio from './Audio';
+
+router.add({
+    target: '#content',
+    rule: '/audio',
+    Component: Audio
+});

frontend/src/audio/ui/AudioPanel.vue

+<template>
+    <v-card hover class="visual-dl-audio">
+        <h3 class="visual-dl-audio-title">{{tagInfo.tag.displayName}}
+            <span class="visual-dl-audio-run-icon">{{tagInfo.run}}</span>
+        </h3>
+        <p>
+            <span>Step:</span>
+            <span>{{audioData.step}}</span>
+            <span class="visual-del-audio-time">{{audioData.wall_time | formatTime}}</span>
+        </p>
+        <v-slider :max="steps"
+                  :min="slider.min"
+                  :step="1"
+                  v-model="currentIndex"
+                  ></v-slider>
+
+        <audio controls>
+            <source src="horse.ogg" type="audio/ogg">
+            Your browser does not support the audio element.
+        </audio>
+    </v-card>
+</template>
+<script>
+import {getPluginAudioAudio} from '../../service';
+
+// the time to refresh chart data
+const intervalTime = 30;
+
+export default {
+    props: ['tagInfo', 'runs', 'running', 'runsItems'],
+    computed: {
+        steps() {
+            let data = this.data || [];
+            return data.length - 1;
+        }
+    },
+    filters: {
+        formatTime: function (value) {
+            if (!value) {
+                return;
+            }
+            // The value was made in seconds, must convert it to milliseconds
+            let time = new Date(value * 1000);
+            var options = {
+                weekday: "short", year: "numeric", month: "short",
+                day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit"
+            };
+            return time.toLocaleDateString("en-US", options);
+        }
+    },
+    data() {
+        return {
+            currentIndex: 0,
+            slider: {
+                value: '0',
+                label: '',
+                min: 0,
+                step: 1
+            },
+            audioData: {},
+            data: [],
+        };
+    },
+    created() {
+        this.getOriginAudioData();
+    },
+    mounted() {
+        if (this.running) {
+            this.startInterval();
+        }
+    },
+
+    beforeDestroy() {
+        this.stopInterval();
+    },
+
+    watch: {
+        running: function (val) {
+            val ? this.startInterval() : this.stopInterval();
+        },
+        currentIndex: function(index) {
+            if (this.data && this.data[index]) {
+                let currentAudioInfo = this.data ? this.data[index] : {};
+                let {query, step, wall_time} = currentAudioInfo;
+                let url = '/data/plugin/images/individualAudio?ts=' + wall_time;
+                let audioSrc = [url, query].join('&');
+                this.audioData = {
+                    audioSrc,
+                    step,
+                    wall_time
+                }
+            }
+        }
+    },
+    methods: {
+        stopInterval() {
+            clearInterval(this.getOringDataInterval);
+        },
+        // get origin data per {{intervalTime}} seconds
+        startInterval() {
+            this.getOringDataInterval = setInterval(() => {
+                this.getOriginAudioData();
+            }, intervalTime * 1000);
+        },
+        getOriginAudioData() {
+            //let {run, tag} = this.tagInfo;
+            let run = this.tagInfo.run
+            let tag = this.tagInfo.tag
+            let {displayName, samples} = tag;
+            let params = {
+                run,
+                tag: displayName,
+                samples
+            };
+            getPluginAudioAudio(params).then(({status, data}) => {
+                if (status === 0) {
+                    this.data = data;
+                    this.currentIndex = data.length - 1;
+                }
+            });
+        }
+    }
+};
+</script>
+<style lang="stylus">
+    .visual-dl-audio
+        font-size 12px
+        width 420px
+        float left
+        margin 20px 30px 10px 0
+        background #fff
+        padding 10px
+        .visual-dl-audio-title
+            font-size 14px
+            line-height 30px
+            .visual-dl-audio-run-icon
+                background #e4e4e4
+                float right
+                margin-right 10px
+                padding 0 10px
+                border solid 1px #e4e4e4
+                border-radius 6px
+                line-height 20px
+                margin-top 4px
+        .visual-del-audio-time
+            float right
+</style>
+

frontend/src/audio/ui/AudioPanelContainer.vue

+<template>
+    <div class="visual-dl-chart-page">
+        <ui-expand-panel :info="total" :title="title">
+            <ui-audio
+                class="visual-dl-chart-audio"
+                v-for="(tagInfo, index) in filteredPageList"
+                :key="index"
+                :tagInfo="tagInfo"
+                :runs="config.runs"
+                :running="config.running"
+                :runsItems="runsItems"
+            ></ui-audio>
+
+            <v-pagination class="visual-dl-sm-pagination"
+                          v-if="total > pageSize"
+                          v-model="currentPage"
+                          :length="pageLength"
+            ></v-pagination>
+        </ui-expand-panel>
+    </div>
+</template>
+<script>
+import ExpandPanel from '../../common/component/ExpandPanel';
+import AudioPanel from './AudioPanel';
+
+import {cloneDeep, flatten} from 'lodash';
+
+export default {
+    props: ['config', 'runsItems', 'tagList', 'title'],
+    components: {
+        'ui-audio': AudioPanel,
+        'ui-expand-panel': ExpandPanel,
+    },
+    computed: {
+        filteredRunsList() {
+            let tagList = this.tagList || [];
+            let runs = this.config.runs || [];
+            let list = cloneDeep(tagList);
+            return flatten(list.slice().map(item => {
+                return item.tagList.filter(one => runs.includes(one.run));
+            }));
+        },
+
+        filteredPageList() {
+            let list = this.filteredRunsList || [];
+            return list.slice((this.currentPage - 1) * this.pageSize, this.currentPage * this.pageSize);
+        },
+        total() {
+            let list = this.filteredRunsList || [];
+            return list.length;
+        },
+        pageLength() {
+            return Math.ceil(this.total / this.pageSize)
+        }
+    },
+    data() {
+        return {
+            // current page
+            currentPage: 1,
+            // item per page
+            pageSize: 8
+        };
+    },
+};
+</script>
+<style lang="stylus">
+@import '~style/variables';
+
++prefix-classes('visual-dl-')
+    .chart-page
+        .audio-chart-box
+            overflow hidden
+            float left
+            .visual-dl-chart-audio
+                float left
+        .audio-chart-box:after
+            content ""
+            clear both
+            display block
+        .sm-pagination
+            height 50px
+            float left
+            width 100%
+</style>
+
+

frontend/src/audio/ui/Config.vue

+<template>
+    <div class="visual-dl-page-config-com">
+        <v-text-field
+            label="Group name RegExp"
+            hint="input a tag group name to search"
+            v-model="config.groupNameReg"
+            dark
+        ></v-text-field>
+
+        <label class="visual-dl-page-checkbox-group-label">Runs</label>
+        <v-checkbox v-for="item in runsItems"
+                    :key="item.name"
+                    :label="item.name"
+                    :value="item.value"
+                    v-model="config.runs"
+                    dark
+        ></v-checkbox>
+
+        <v-btn :color="config.running ? 'primary' : 'error'"
+                  v-model="config.running"
+                  @click="toggleAllRuns"
+                  class="visual-dl-page-run-toggle"
+                  dark block
+        >
+            {{config.running ? 'Running' : 'Stopped'}}
+        </v-btn>
+    </div>
+</template>
+<script>
+
+export default {
+    props: {
+        runsItems: Array,
+        config: Object
+    },
+    data() {
+        return {
+        };
+    },
+    methods: {
+        toggleAllRuns() {
+            this.config.running = !this.config.running;
+        }
+    }
+};
+
+</script>
+<style lang="stylus">
++prefix-classes('visual-dl-page-')
+    .config-com
+        padding 20px
+        .run-toggle
+            margin-top 20px
+        .checkbox-group-label
+            display flex
+            margin-top 20px
+            margin-bottom 10px
+</style>

frontend/src/common/component/AppMenu.vue

@@ -32,6 +32,11 @@ export default {
                 title: 'IMAGES',
                 name: 'images'
             },
+            {
+                url: '/audio',
+                title: 'AUDIO',
+                name: 'audio'
+            },
             {
                 url: '/histograms',
                 title: 'HISTOGRAMS',

frontend/src/images/ui/Image.vue

@@ -153,10 +153,6 @@ export default {
                 border-radius 6px
                 line-height 20px
                 margin-top 4px
-        .visual-dl-chart-actions
-            .sm-form-item
-                width 300px
-                display inline-block
         .visual-del-image-time
             float right
 </style>

frontend/src/router/index.js

@@ -6,6 +6,7 @@ import Histogram from '@/histogram/Histogram'
 import Images from '@/images/Images'
 import Graph from '@/graph/Graph'
 import Texts from '@/texts/Texts'
+import Audio from '@/audio/Audio'
 
 Vue.use(Router)
 
@@ -35,6 +36,11 @@ export default new Router({
       path: '/texts',
       name: 'Texts',
       component: Texts
+    },
+    {
+      path: '/audio',
+      name: 'Audio',
+      component: Audio
     }
   ]
 })

frontend/src/service.js

@@ -19,3 +19,7 @@ export const getPluginGraphsGraph = makeService('/data/plugin/graphs/graph');
 export const getPluginTextsTags = makeService('/data/plugin/texts/tags');
 
 export const getPluginTextsTexts = makeService('/data/plugin/texts/texts');
+
+export const getPluginAudioTags = makeService('/data/plugin/audio/tags');
+
+export const getPluginAudioAudio = makeService('/data/plugin/audio/audio');