Remove unused scratch buffer for integer mode LSTM (#1178)

* Remove unused scratch buffer for integer mode LSTM

* Revert changes to lstm_eval
Co-authored-by: Nmergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

tensorflow/lite/micro/kernels/lstm_eval_test.cc

@@ -774,7 +774,7 @@ TF_LITE_MICRO_TEST(TestOneFullyQuantizedLSTM) {
       /*time_major=*/true, param, 50, activation, cell, output,
       one_parameter.GetScratch0(), one_parameter.GetScratch1(),
       one_parameter.GetScratch2(), one_parameter.GetScratch3(),
-      one_parameter.GetScratch4(), one_parameter.GetScratch5());
+      one_parameter.GetScratch4(), nullptr);
 
   // Verify results.
   const int16_t expected_cell[20] = {

tensorflow/lite/micro/kernels/unidirectional_sequence_lstm.cc

@@ -1406,18 +1406,14 @@ TfLiteStatus UnidirectionalSequenceLstmPrepare(TfLiteContext* context,
     // Populate quantization parameters.
     PopulateQuantizedLstmParams8x8_16(context, node,
                                       &op_data->integer_lstm_param);
-    // Allocate scratch buffer. Need 6 16bit buffer with size n_batch * n_cell
-    // and 1 8bit buffer with size n_batch * n_cell. We also need 1 32 bit
-    // buffer with size n_batch * n_cell.
-    //
-    // Handle cifg case as well, which might save one buffer.
-    for (int i = 0; i < 6; ++i) {
+    // Allocate scratch buffer. Need 4 16-bit buffer with size n_batch * n_cell
+    // and 1 8-bit buffer with size n_batch * n_cell. For integer
+    // UnidirectionalSequenceLSTM, we do not need the extra 32-bit buffer.
+    for (int i = 0; i < 5; ++i) {
       TfLiteType buffer_type = kTfLiteInt16;
 
       if (i == 4) {
         buffer_type = kTfLiteInt8;
-      } else if (i == 5) {
-        buffer_type = kTfLiteInt32;
       }
 
       TF_LITE_ENSURE_OK(
@@ -1679,8 +1675,7 @@ TfLiteStatus UnidirectionalSequenceLstmEval(TfLiteContext* context,
                 context->GetScratchBuffer(context, op_data->scratch_index[3])),
             reinterpret_cast<int8_t*>(
                 context->GetScratchBuffer(context, op_data->scratch_index[4])),
-            reinterpret_cast<int32_t*>(
-                context->GetScratchBuffer(context, op_data->scratch_index[5])));
+            nullptr);
       }
     } break;
     default: