Merge pull request #1649 from SmileGoat/update_release_note

[speechx]refactor linear feature:unify vector & remove redundant function & add remained_wav cache shift wav

speechx/examples/feat/linear_spectrogram_main.cc

@@ -181,6 +181,10 @@ int main(int argc, char* argv[]) {
     ppspeech::LinearSpectrogramOptions opt;
     opt.frame_opts.frame_length_ms = 20;
     opt.frame_opts.frame_shift_ms = 10;
+    opt.frame_opts.dither = 0.0;
+    opt.frame_opts.remove_dc_offset = false;
+    opt.frame_opts.window_type = "hanning";
+    opt.frame_opts.preemph_coeff = 0.0;
     LOG(INFO) << "frame length (ms): " << opt.frame_opts.frame_length_ms;
     LOG(INFO) << "frame shift (ms): " << opt.frame_opts.frame_shift_ms;
 

speechx/speechx/frontend/audio/linear_spectrogram.cc

@@ -14,6 +14,8 @@
 
 #include "frontend/audio/linear_spectrogram.h"
 #include "kaldi/base/kaldi-math.h"
+#include "kaldi/feat/feature-common.h"
+#include "kaldi/feat/feature-functions.h"
 #include "kaldi/matrix/matrix-functions.h"
 
 namespace ppspeech {
@@ -21,30 +23,23 @@ namespace ppspeech {
 using kaldi::int32;
 using kaldi::BaseFloat;
 using kaldi::Vector;
+using kaldi::SubVector;
 using kaldi::VectorBase;
 using kaldi::Matrix;
 using std::vector;
 
 LinearSpectrogram::LinearSpectrogram(
     const LinearSpectrogramOptions& opts,
-    std::unique_ptr<FrontendInterface> base_extractor) {
-    opts_ = opts;
+    std::unique_ptr<FrontendInterface> base_extractor)
+    : opts_(opts), feature_window_funtion_(opts.frame_opts) {
     base_extractor_ = std::move(base_extractor);
     int32 window_size = opts.frame_opts.WindowSize();
     int32 window_shift = opts.frame_opts.WindowShift();
-    fft_points_ = window_size;
+    dim_ = window_size / 2 + 1;
     chunk_sample_size_ =
         static_cast<int32>(opts.streaming_chunk * opts.frame_opts.samp_freq);
-    hanning_window_.resize(window_size);
-
-    double a = M_2PI / (window_size - 1);
-    hanning_window_energy_ = 0;
-    for (int i = 0; i < window_size; ++i) {
-        hanning_window_[i] = 0.5 - 0.5 * cos(a * i);
-        hanning_window_energy_ += hanning_window_[i] * hanning_window_[i];
-    }
-
-    dim_ = fft_points_ / 2 + 1;  // the dimension is Fs/2 Hz
+    hanning_window_energy_ = kaldi::VecVec(feature_window_funtion_.window,
+                                           feature_window_funtion_.window);
 }
 
 void LinearSpectrogram::Accept(const VectorBase<BaseFloat>& inputs) {
@@ -56,99 +51,57 @@ bool LinearSpectrogram::Read(Vector<BaseFloat>* feats) {
     bool flag = base_extractor_->Read(&input_feats);
     if (flag == false || input_feats.Dim() == 0) return false;
 
-    vector<BaseFloat> input_feats_vec(input_feats.Dim());
-    std::memcpy(input_feats_vec.data(),
-                input_feats.Data(),
-                input_feats.Dim() * sizeof(BaseFloat));
-    vector<vector<BaseFloat>> result;
-    Compute(input_feats_vec, result);
-    int32 feat_size = 0;
-    if (result.size() != 0) {
-        feat_size = result.size() * result[0].size();
-    }
-    feats->Resize(feat_size);
-    // todo refactor (SimleGoat)
-    for (size_t idx = 0; idx < feat_size; ++idx) {
-        (*feats)(idx) = result[idx / dim_][idx % dim_];
-    }
-    return true;
-}
-
-void LinearSpectrogram::Hanning(vector<float>* data) const {
-    CHECK_GE(data->size(), hanning_window_.size());
-
-    for (size_t i = 0; i < hanning_window_.size(); ++i) {
-        data->at(i) *= hanning_window_[i];
-    }
-}
-
-bool LinearSpectrogram::NumpyFft(vector<BaseFloat>* v,
-                                 vector<BaseFloat>* real,
-                                 vector<BaseFloat>* img) const {
-    Vector<BaseFloat> v_tmp;
-    v_tmp.Resize(v->size());
-    std::memcpy(v_tmp.Data(), v->data(), sizeof(BaseFloat) * (v->size()));
-    RealFft(&v_tmp, true);
-    v->resize(v_tmp.Dim());
-    std::memcpy(v->data(), v_tmp.Data(), sizeof(BaseFloat) * (v->size()));
-
-    real->push_back(v->at(0));
-    img->push_back(0);
-    for (int i = 1; i < v->size() / 2; i++) {
-        real->push_back(v->at(2 * i));
-        img->push_back(v->at(2 * i + 1));
-    }
-    real->push_back(v->at(1));
-    img->push_back(0);
-
+    int32 feat_len = input_feats.Dim();
+    int32 left_len = reminded_wav_.Dim();
+    Vector<BaseFloat> waves(feat_len + left_len);
+    waves.Range(0, left_len).CopyFromVec(reminded_wav_);
+    waves.Range(left_len, feat_len).CopyFromVec(input_feats);
+    Compute(waves, feats);
+    int32 frame_shift = opts_.frame_opts.WindowShift();
+    int32 num_frames = kaldi::NumFrames(waves.Dim(), opts_.frame_opts);
+    int32 left_samples = waves.Dim() - frame_shift * num_frames;
+    reminded_wav_.Resize(left_samples);
+    reminded_wav_.CopyFromVec(
+        waves.Range(frame_shift * num_frames, left_samples));
     return true;
 }
 
 // Compute spectrogram feat
-// todo: refactor later (SmileGoat)
-bool LinearSpectrogram::Compute(const vector<float>& waves,
-                                vector<vector<float>>& feats) {
-    int num_samples = waves.size();
-    const int& frame_length = opts_.frame_opts.WindowSize();
-    const int& sample_rate = opts_.frame_opts.samp_freq;
-    const int& frame_shift = opts_.frame_opts.WindowShift();
-    const int& fft_points = fft_points_;
-    const float scale = hanning_window_energy_ * sample_rate;
+bool LinearSpectrogram::Compute(const Vector<BaseFloat>& waves,
+                                Vector<BaseFloat>* feats) {
+    int32 num_samples = waves.Dim();
+    int32 frame_length = opts_.frame_opts.WindowSize();
+    int32 sample_rate = opts_.frame_opts.samp_freq;
+    BaseFloat scale = 2.0 / (hanning_window_energy_ * sample_rate);
 
     if (num_samples < frame_length) {
         return true;
     }
 
-    int num_frames = 1 + ((num_samples - frame_length) / frame_shift);
-    feats.resize(num_frames);
-    vector<float> fft_real((fft_points_ / 2 + 1), 0);
-    vector<float> fft_img((fft_points_ / 2 + 1), 0);
-    vector<float> v(frame_length, 0);
-    vector<float> power((fft_points / 2 + 1));
-
-    for (int i = 0; i < num_frames; ++i) {
-        vector<float> data(waves.data() + i * frame_shift,
-                           waves.data() + i * frame_shift + frame_length);
-        Hanning(&data);
-        fft_img.clear();
-        fft_real.clear();
-        v.assign(data.begin(), data.end());
-        NumpyFft(&v, &fft_real, &fft_img);
-
-        feats[i].resize(fft_points / 2 + 1);  // the last dimension is Fs/2 Hz
-        for (int j = 0; j < (fft_points / 2 + 1); ++j) {
-            power[j] = fft_real[j] * fft_real[j] + fft_img[j] * fft_img[j];
-            feats[i][j] = power[j];
-
-            if (j == 0 || j == feats[0].size() - 1) {
-                feats[i][j] /= scale;
-            } else {
-                feats[i][j] *= (2.0 / scale);
-            }
-
-            // log added eps=1e-14
-            feats[i][j] = std::log(feats[i][j] + 1e-14);
-        }
+    int32 num_frames = kaldi::NumFrames(num_samples, opts_.frame_opts);
+    feats->Resize(num_frames * dim_);
+    Vector<BaseFloat> window;
+
+    for (int frame_idx = 0; frame_idx < num_frames; ++frame_idx) {
+        kaldi::ExtractWindow(0,
+                             waves,
+                             frame_idx,
+                             opts_.frame_opts,
+                             feature_window_funtion_,
+                             &window,
+                             NULL);
+
+        SubVector<BaseFloat> output_row(feats->Data() + frame_idx * dim_, dim_);
+        window.Resize(frame_length, kaldi::kCopyData);
+        RealFft(&window, true);
+        kaldi::ComputePowerSpectrum(&window);
+        SubVector<BaseFloat> power_spectrum(window, 0, dim_);
+        power_spectrum.Scale(scale);
+        power_spectrum(0) = power_spectrum(0) / 2;
+        power_spectrum(dim_ - 1) = power_spectrum(dim_ - 1) / 2;
+        power_spectrum.Add(1e-14);
+        power_spectrum.ApplyLog();
+        output_row.CopyFromVec(power_spectrum);
     }
     return true;
 }

speechx/speechx/frontend/audio/linear_spectrogram.h

@@ -49,19 +49,15 @@ class LinearSpectrogram : public FrontendInterface {
     virtual void Reset() { base_extractor_->Reset(); }
 
   private:
-    void Hanning(std::vector<kaldi::BaseFloat>* data) const;
-    bool Compute(const std::vector<kaldi::BaseFloat>& waves,
-                 std::vector<std::vector<kaldi::BaseFloat>>& feats);
-    bool NumpyFft(std::vector<kaldi::BaseFloat>* v,
-                  std::vector<kaldi::BaseFloat>* real,
-                  std::vector<kaldi::BaseFloat>* img) const;
+    bool Compute(const kaldi::Vector<kaldi::BaseFloat>& waves,
+                 kaldi::Vector<kaldi::BaseFloat>* feats);
 
-    kaldi::int32 fft_points_;
     size_t dim_;
-    std::vector<kaldi::BaseFloat> hanning_window_;
+    kaldi::FeatureWindowFunction feature_window_funtion_;
     kaldi::BaseFloat hanning_window_energy_;
     LinearSpectrogramOptions opts_;
     std::unique_ptr<FrontendInterface> base_extractor_;
+    kaldi::Vector<kaldi::BaseFloat> reminded_wav_;
     int chunk_sample_size_;
     DISALLOW_COPY_AND_ASSIGN(LinearSpectrogram);
 };