Make metrics exporter still work even when storage layer failed (#7041)

CHANGES.md

@@ -32,7 +32,7 @@ Release Notes.
 * Introduce method interceptor API v2
 * Fix ClassCast issue for RequestHolder/ResponseHolder.
 * fixed `jdk-threading-plugin` memory leak.
-* Optimize multiple field reflection opeartion in Fiegn plugin.
+* Optimize multiple field reflection operation in Feign plugin.
 
 #### OAP-Backend
 * BugFix: filter invalid Envoy access logs whose socket address is empty.
@@ -57,6 +57,7 @@ Release Notes.
 * Events can be configured as alarm source.
 * Make the number of core worker in meter converter thread pool configurable.
 * Add HTTP implementation of logs reporting protocol.
+* Make metrics exporter still work even when storage layer failed.
 
 #### UI
 * Add logo for kong plugin.

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/worker/MetricsPersistentWorker.java

@@ -18,7 +18,6 @@
 
 package org.apache.skywalking.oap.server.core.analysis.worker;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -209,16 +208,23 @@ public class MetricsPersistentWorker extends PersistenceWorker<Metrics> {
     /**
      * Load data from the storage, if {@link #enableDatabaseSession} == true, only load data when the id doesn't exist.
      */
-    private void loadFromStorage(List<Metrics> metrics) throws IOException {
-        if (!enableDatabaseSession) {
-            context.clear();
-        }
+    private void loadFromStorage(List<Metrics> metrics) {
+        try {
+            List<Metrics> noInCacheMetrics = metrics.stream()
+                                                    .filter(m -> !context.containsKey(m) || !enableDatabaseSession)
+                                                    .collect(Collectors.toList());
+            if (noInCacheMetrics.isEmpty()) {
+                return;
+            }
 
-        List<Metrics> noInCacheMetrics = metrics.stream()
-                                                .filter(m -> !context.containsKey(m))
-                                                .collect(Collectors.toList());
-        if (!noInCacheMetrics.isEmpty()) {
-            metricsDAO.multiGet(model, noInCacheMetrics).forEach(m -> context.put(m, m));
+            final List<Metrics> dbMetrics = metricsDAO.multiGet(model, noInCacheMetrics);
+            if (!enableDatabaseSession) {
+                // Clear the cache only after results from DB are returned successfully.
+                context.clear();
+            }
+            dbMetrics.forEach(m -> context.put(m, m));
+        } catch (final Exception e) {
+            log.error("Failed to load metrics for merging", e);
         }
     }