Audio integration to front end (#360)

* Audio integration to front end
* Add flask end point
* Read and decode audio data

* refract a typo

* fix clang format

frontend/src/audio/ui/AudioPanel.vue

@@ -14,8 +14,7 @@
                   v-model="currentIndex"
                   ></v-slider>
 
-        <audio controls>
-            <source src="horse.ogg" type="audio/ogg">
+        <audio controls :src="audioData.audioSrc">
             Your browser does not support the audio element.
         </audio>
     </v-card>
@@ -82,7 +81,7 @@ export default {
             if (this.data && this.data[index]) {
                 let currentAudioInfo = this.data ? this.data[index] : {};
                 let {query, step, wall_time} = currentAudioInfo;
-                let url = '/data/plugin/images/individualAudio?ts=' + wall_time;
+                let url = '/data/plugin/audio/individualAudio?ts=' + wall_time;
                 let audioSrc = [url, query].join('&');
                 this.audioData = {
                     audioSrc,

frontend/tool/webpack.config.js

@@ -92,7 +92,7 @@ const config = {
                 use: getLoaders(isDev, 'stylus')
             },
             {
-                test: /\.(gif|png|jpe?g|svg)$/i,
+                test: /\.(gif|png|jpe?g|svg|wav)$/i,
                 loader: 'file-loader',
                 options: {
                     name: 'assets/[name].[hash].[ext]'

visualdl/server/lib.py

@@ -6,6 +6,7 @@ from tempfile import NamedTemporaryFile
 import numpy as np
 from PIL import Image
 from .log import logger
+import wave
 
 try:
     from urllib.parse import urlencode
@@ -155,6 +156,86 @@ def get_invididual_image(storage, mode, tag, step_index, max_size=80):
         return tempfile
 
 
+def get_audio_tags(storage):
+    result = {}
+
+    for mode in storage.modes():
+        with storage.mode(mode) as reader:
+            tags = reader.tags('audio')
+            if tags:
+                result[mode] = {}
+                for tag in tags:
+                    audio = reader.audio(tag)
+                    for i in range(max(1, audio.num_samples())):
+                        caption = tag if audio.num_samples(
+                        ) <= 1 else '%s/%d' % (tag, i)
+                        result[mode][caption] = {
+                            'displayName': caption,
+                            'description': "",
+                            'samples': 1,
+                        }
+
+    return result
+
+
+def get_audio_tag_steps(storage, mode, tag):
+    # remove suffix '/x'
+    res = re.search(r".*/([0-9]+$)", tag)
+    sample_index = 0
+    origin_tag = tag
+    if res:
+        tag = tag[:tag.rfind('/')]
+        sample_index = int(res.groups()[0])
+
+    with storage.mode(mode) as reader:
+        audio = reader.audio(tag)
+        res = []
+
+    for step_index in range(audio.num_records()):
+        record = audio.record(step_index, sample_index)
+
+        query = urlencode({
+            'sample': 0,
+            'index': step_index,
+            'tag': origin_tag,
+            'run': mode,
+        })
+        res.append({
+            'step': record.step_id(),
+            'wall_time': audio.timestamp(step_index),
+            'query': query,
+        })
+
+    return res
+
+
+def get_individual_audio(storage, mode, tag, step_index, max_size=80):
+
+    with storage.mode(mode) as reader:
+        res = re.search(r".*/([0-9]+$)", tag)
+        # remove suffix '/x'
+        offset = 0
+        if res:
+            offset = int(res.groups()[0])
+            tag = tag[:tag.rfind('/')]
+
+        audio = reader.audio(tag)
+        record = audio.record(step_index, offset)
+
+        data = np.array(record.data(), dtype='uint8')
+
+        tempfile = NamedTemporaryFile(mode='w+b', suffix='.wav')
+
+        wavfile = wave.open(tempfile, 'wb')
+        wavfile.setnchannels(2)
+        wavfile.setsampwidth(2)
+        wavfile.writeframes(data.tostring())
+
+        tempfile.seek(0, 0)
+
+        return tempfile
+
+
 def get_histogram_tags(storage):
     return get_tags(storage, 'histogram')
 

visualdl/server/visualDL

@@ -163,6 +163,14 @@ def image_tags():
     return Response(json.dumps(result), mimetype='application/json')
 
 
+@app.route("/data/plugin/audio/tags")
+def audio_tags():
+    data = cache_get("/data/plugin/audio/tags", try_call, lib.get_audio_tags,
+                     log_reader)
+    result = gen_result(0, "", data)
+    return Response(json.dumps(result), mimetype='application/json')
+
+
 @app.route("/data/plugin/histograms/tags")
 def histogram_tags():
     data = cache_get("/data/plugin/histograms/tags", try_call,
@@ -245,6 +253,35 @@ def embeddings():
     result = gen_result(0, "", data)
     return Response(json.dumps(result), mimetype='application/json')
 
+@app.route('/data/plugin/audio/audio')
+def audio():
+    mode = request.args.get('run')
+    tag = request.args.get('tag')
+    key = os.path.join('/data/plugin/audio/audio', mode, tag)
+
+    data = cache_get(key, try_call, lib.get_audio_tag_steps, log_reader, mode,
+                     tag)
+    result = gen_result(0, "", data)
+
+    return Response(json.dumps(result), mimetype='application/json')
+
+
+@app.route('/data/plugin/audio/individualAudio')
+def individual_audio():
+    mode = request.args.get('run')
+    print mode
+    tag = request.args.get('tag')  # include a index
+    step_index = int(request.args.get('index'))  # index of step
+
+    key = os.path.join('/data/plugin/audio/individualAudio', mode, tag,
+                       str(step_index))
+    data = cache_get(key, try_call, lib.get_individual_audio, log_reader, mode,
+                     tag, step_index)
+    response = send_file(
+        data, as_attachment=True, attachment_filename='audio.wav')
+    return response
+
+
 @app.route('/data/plugin/graphs/graph')
 def graph():
     # TODO(ChunweiYan) need to add a config for whether have graph.