Dreamview: publish audio event in backend

modules/common/adapters/adapter_gflags.cc

@@ -210,6 +210,8 @@ DEFINE_string(v2x_trafficlight_topic, "/apollo/v2x/traffic_light",
               "v2x trafficlight topic name");
 DEFINE_string(storytelling_topic, "/apollo/storytelling",
               "Storytelling topic.");
+DEFINE_string(audio_event_topic, "/apollo/audio_event",
+              "Audio event topic.");
 
 DEFINE_string(guardian_topic, "/apollo/guardian", "Guardian topic.");
 DEFINE_string(gnss_raw_data_topic, "/apollo/sensor/gnss/raw_data",

modules/common/adapters/adapter_gflags.h

@@ -109,6 +109,7 @@ DECLARE_string(gnss_raw_data_topic);
 DECLARE_string(stream_status_topic);
 DECLARE_string(heading_topic);
 DECLARE_string(rtcm_data_topic);
+DECLARE_string(audio_event_topic);
 
 // Guardian topic
 DECLARE_string(guardian_topic);

modules/dreamview/backend/hmi/BUILD

@@ -67,6 +67,7 @@ cc_library(
         ":vehicle_manager",
         "//cyber",
         "//cyber/proto:dag_conf_cc_proto",
+        "//modules/audio/proto:audio_event_cc_proto",
         "//modules/common/adapters:adapter_gflags",
         "//modules/common/configs:config_gflags",
         "//modules/common/kv_db",
@@ -78,6 +79,7 @@ cc_library(
         "//modules/dreamview/proto:hmi_config_cc_proto",
         "//modules/dreamview/proto:hmi_mode_cc_proto",
         "//modules/dreamview/proto:hmi_status_cc_proto",
+        "//modules/localization/proto:localization_cc_proto",
         "//modules/monitor/proto:system_status_cc_proto",
         "@boost",
         "@com_google_absl//absl/strings",

modules/dreamview/backend/hmi/hmi.cc

@@ -116,6 +116,34 @@ void HMI::RegisterMessageHandlers() {
         }
       });
 
+  // HMI client asks for adding new AudioEvent.
+  websocket_->RegisterMessageHandler(
+      "SubmitAudioEvent",
+      [this](const Json& json, WebSocketHandler::Connection* conn) {
+        // json should contain event_time_ms, obstacle_id, audio_type,
+        // moving_result, audio_direction and is_siren_on.
+        uint64_t event_time_ms;
+        int obstacle_id;
+        int audio_type;
+        int moving_result;
+        int audio_direction;
+        bool is_siren_on;
+        if (JsonUtil::GetNumber(json, "event_time_ms", &event_time_ms) &&
+            JsonUtil::GetNumber(json, "obstacle_id", &obstacle_id) &&
+            JsonUtil::GetNumber(json, "audio_type", &audio_type) &&
+            JsonUtil::GetNumber(json, "moving_result", &moving_result) &&
+            JsonUtil::GetNumber(json, "audio_direction", &audio_direction) &&
+            JsonUtil::GetBoolean(json, "is_siren_on", &is_siren_on)) {
+          hmi_worker_->SubmitAudioEvent(event_time_ms, obstacle_id, audio_type,
+                                        moving_result, audio_direction,
+                                        is_siren_on);
+          monitor_log_buffer_.INFO("Audio event added.");
+        } else {
+          AERROR << "Truncated SubmitAudioEvent request.";
+          monitor_log_buffer_.WARN("Failed to submit an audio event.");
+        }
+      });
+
   // HMI client asks for adding new DriveEvent.
   websocket_->RegisterMessageHandler(
       "SubmitDriveEvent",

modules/dreamview/backend/hmi/hmi_worker.cc

@@ -52,6 +52,7 @@ namespace apollo {
 namespace dreamview {
 namespace {
 
+using apollo::audio::AudioEvent;
 using apollo::canbus::Chassis;
 using apollo::common::DriveEvent;
 using apollo::common::KVDB;
@@ -59,6 +60,7 @@ using apollo::control::DrivingAction;
 using apollo::cyber::Clock;
 using apollo::cyber::Node;
 using apollo::cyber::proto::DagConfig;
+using apollo::localization::LocalizationEstimate;
 using apollo::monitor::ComponentStatus;
 using apollo::monitor::SystemStatus;
 using google::protobuf::Map;
@@ -275,6 +277,8 @@ void HMIWorker::InitStatus() {
 void HMIWorker::InitReadersAndWriters() {
   status_writer_ = node_->CreateWriter<HMIStatus>(FLAGS_hmi_status_topic);
   pad_writer_ = node_->CreateWriter<control::PadMessage>(FLAGS_pad_topic);
+  audio_event_writer_ =
+      node_->CreateWriter<AudioEvent>(FLAGS_audio_event_topic);
   drive_event_writer_ =
       node_->CreateWriter<DriveEvent>(FLAGS_drive_event_topic);
 
@@ -319,6 +323,8 @@ void HMIWorker::InitReadersAndWriters() {
         }
       });
 
+  localization_reader_ =
+      node_->CreateReader<LocalizationEstimate>(FLAGS_localization_topic);
   // Received Chassis, trigger action if there is high beam signal.
   chassis_reader_ = node_->CreateReader<Chassis>(
       FLAGS_chassis_topic, [this](const std::shared_ptr<Chassis>& chassis) {
@@ -381,6 +387,43 @@ bool HMIWorker::Trigger(const HMIAction action, const std::string& value) {
   return true;
 }
 
+void HMIWorker::SubmitAudioEvent(const uint64_t event_time_ms,
+                                 const int obstacle_id, const int audio_type,
+                                 const int moving_result,
+                                 const int audio_direction,
+                                 const bool is_siren_on) {
+  std::shared_ptr<AudioEvent> audio_event = std::make_shared<AudioEvent>();
+  apollo::common::util::FillHeader("HMI", audio_event.get());
+  // Here we reuse the header time field as the event occurring time.
+  // A better solution might be adding an event time field to DriveEvent proto
+  // to make it clear.
+  audio_event->mutable_header()->set_timestamp_sec(
+      static_cast<double>(event_time_ms) / 1000.0);
+  audio_event->set_id(obstacle_id);
+  audio_event->set_audio_type(
+      static_cast<apollo::audio::AudioType>(audio_type));
+  audio_event->set_moving_result(
+      static_cast<apollo::audio::MovingResult>(moving_result));
+  audio_event->set_audio_direction(
+      static_cast<apollo::audio::AudioDirection>(audio_direction));
+  audio_event->set_siren_is_on(is_siren_on);
+
+  // Read the current localization pose
+  localization_reader_->Observe();
+  if (localization_reader_->Empty()) {
+    AERROR << "Failed to get localization associated with the audio event: "
+           << audio_event->DebugString() << "\n Localization reader is empty!";
+    return;
+  }
+
+  const std::shared_ptr<LocalizationEstimate> localization =
+      localization_reader_->GetLatestObserved();
+  audio_event->mutable_pose()->CopyFrom(localization->pose());
+  AINFO << "AudioEvent: " << audio_event->DebugString();
+
+  audio_event_writer_->Write(audio_event);
+}
+
 void HMIWorker::SubmitDriveEvent(const uint64_t event_time_ms,
                                  const std::string& event_msg,
                                  const std::vector<std::string>& event_types,

modules/dreamview/backend/hmi/hmi_worker.h

@@ -26,12 +26,14 @@
 
 #include "cyber/cyber.h"
 #include "cyber/time/time.h"
+#include "modules/audio/proto/audio_event.pb.h"
 #include "modules/canbus/proto/chassis.pb.h"
 #include "modules/common/proto/drive_event.pb.h"
 #include "modules/control/proto/pad_msg.pb.h"
 #include "modules/dreamview/proto/hmi_config.pb.h"
 #include "modules/dreamview/proto/hmi_mode.pb.h"
 #include "modules/dreamview/proto/hmi_status.pb.h"
+#include "modules/localization/proto/localization.pb.h"
 
 /**
  * @namespace apollo::dreamview
@@ -61,6 +63,11 @@ class HMIWorker {
     status_update_handlers_.push_back(handler);
   }
 
+  // Submit an AudioEvent
+  void SubmitAudioEvent(const uint64_t event_time_ms, const int obstacle_id,
+                        const int audio_type, const int moving_result,
+                        const int audio_direction, const bool is_siren_on);
+
   // Submit a DriveEvent.
   void SubmitDriveEvent(const uint64_t event_time_ms,
                         const std::string& event_msg,
@@ -113,8 +120,12 @@ class HMIWorker {
   // Cyber members.
   std::shared_ptr<apollo::cyber::Node> node_;
   std::shared_ptr<cyber::Reader<apollo::canbus::Chassis>> chassis_reader_;
+  std::shared_ptr<cyber::Reader<apollo::localization::LocalizationEstimate>>
+      localization_reader_;
   std::shared_ptr<cyber::Writer<HMIStatus>> status_writer_;
   std::shared_ptr<cyber::Writer<apollo::control::PadMessage>> pad_writer_;
+  std::shared_ptr<cyber::Writer<apollo::audio::AudioEvent>>
+      audio_event_writer_;
   std::shared_ptr<cyber::Writer<apollo::common::DriveEvent>>
       drive_event_writer_;
 };

modules/dreamview/backend/simulation_world/BUILD

@@ -10,6 +10,7 @@ cc_library(
     copts = ['-DMODULE_NAME=\\"dreamview\\"'],
     deps = [
         "//cyber",
+        "//modules/audio/proto:audio_event_cc_proto",
         "//modules/canbus/proto:chassis_cc_proto",
         "//modules/common/adapters:adapter_gflags",
         "//modules/common/configs:vehicle_config_helper",

modules/dreamview/backend/simulation_world/simulation_world_service.cc

@@ -38,6 +38,7 @@
 namespace apollo {
 namespace dreamview {
 
+using apollo::audio::AudioEvent;
 using apollo::canbus::Chassis;
 using apollo::common::DriveEvent;
 using apollo::common::PathPoint;
@@ -281,6 +282,13 @@ void SimulationWorldService::InitReaders() {
   relative_map_reader_ = node_->CreateReader<MapMsg>(FLAGS_relative_map_topic);
   storytelling_reader_ = node_->CreateReader<Stories>(FLAGS_storytelling_topic);
 
+  audio_event_reader_ = node_->CreateReader<AudioEvent>(
+      FLAGS_audio_event_topic,
+      [this](const std::shared_ptr<AudioEvent> &audio_event) {
+        this->PublishMonitorMessage(
+            MonitorMessageItem::WARN,
+            apollo::audio::AudioType_Name(audio_event->audio_type()));
+      });
   drive_event_reader_ = node_->CreateReader<DriveEvent>(
       FLAGS_drive_event_topic,
       [this](const std::shared_ptr<DriveEvent> &drive_event) {

modules/dreamview/backend/simulation_world/simulation_world_service.h

@@ -35,6 +35,7 @@
 #include "gtest/gtest_prod.h"
 #include "nlohmann/json.hpp"
 
+#include "modules/audio/proto/audio_event.pb.h"
 #include "modules/common/monitor_log/monitor_log_buffer.h"
 #include "modules/common/proto/drive_event.pb.h"
 #include "modules/common/proto/pnc_point.pb.h"
@@ -367,6 +368,8 @@ class SimulationWorldService {
       navigation_reader_;
   std::shared_ptr<cyber::Reader<apollo::relative_map::MapMsg>>
       relative_map_reader_;
+  std::shared_ptr<cyber::Reader<apollo::audio::AudioEvent>>
+      audio_event_reader_;
   std::shared_ptr<cyber::Reader<apollo::common::DriveEvent>>
       drive_event_reader_;
   std::shared_ptr<cyber::Reader<apollo::common::monitor::MonitorMessage>>