Fix Elasticsearch storage: In the merged index, column's property `indexOnly`...

Fix Elasticsearch storage: In the merged index, column's property `indexOnly` not applied and cannot be updated. (#10060)

docs/en/changes/changes.md

@@ -137,6 +137,7 @@
 * Rename `BanyanDB.ShardingKey` to `BanyanDB.SeriesID`.
 * Self-Observability: Add counters for metrics reading from DB or cached. Dashboard:`Metrics Persistent Cache Count`.
 * Self-Observability: Fix `GC Time` calculation.
+* Fix Elasticsearch storage: In `No-Sharding Mode`, column's property `indexOnly` not applied and cannot be updated.
 * Update the `trace_id` field as storage only(cannot be queried) in `top_n_database_statement`, `top_n_cache_read_command`, `top_n_cache_read_command` index.
 
 #### UI

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructures.java

@@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
 import lombok.EqualsAndHashCode;
 import org.apache.skywalking.library.elasticsearch.response.Mappings;
@@ -143,17 +144,36 @@ public class IndexStructures {
             if (!isContains) {
                 return false;
             }
-            return fields.source.getExcludes().containsAll(this.source.getExcludes());
+            Set<String> inputExcludes = fields.source.getExcludes();
+            Set<String> excludes = source.getExcludes();
+            //need to add new excludes
+            if (!excludes.containsAll(inputExcludes)) {
+                return false;
+            }
+            //need to delete existing excludes
+            for (String p : fields.properties.keySet()) {
+                if (!inputExcludes.contains(p) && excludes.contains(p)) {
+                    return false;
+                }
+            }
+            return true;
         }
 
         /**
          * Append new fields and update.
          * Properties combine input and exist, update property's attribute, won't remove old one.
-         * Source will be updated to the input.
+         * If the existed `excludes` contains a not existing property, the excluded field would be removed.
          */
         private void appendNewFields(Fields fields) {
             properties.putAll(fields.properties);
-            source = fields.source;
+            Set<String> inputExcludes = fields.source.getExcludes();
+            Set<String> excludes = source.getExcludes();
+            excludes.addAll(inputExcludes);
+            fields.properties.keySet().forEach(p -> {
+                if (!inputExcludes.contains(p) && excludes.contains(p)) {
+                    excludes.remove(p);
+                }
+            });
         }
 
         /**

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexStructuresTest.java

@@ -74,6 +74,7 @@ public class IndexStructuresTest {
             "test", Mappings.builder()
                             .type(ElasticSearchClient.TYPE)
                             .properties(properties)
+                            .source(new Mappings.Source())
                             .build(), new HashMap<>());
         Mappings mapping = structures.getMapping("test");
         Assert.assertEquals(properties, mapping.getProperties());
@@ -84,6 +85,7 @@ public class IndexStructuresTest {
             "test", Mappings.builder()
                             .type(ElasticSearchClient.TYPE)
                             .properties(properties2)
+                            .source(new Mappings.Source())
                             .build(), new HashMap<>());
         mapping = structures.getMapping("test");
         HashMap<String, Object> res = new HashMap<>();

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/MockEsInstallTest.java

@@ -127,8 +127,8 @@ public class MockEsInstallTest {
                         .source(new Mappings.Source())
                     .build(),
                 new HashSet<>(ImmutableSet.of("a", "b")),
-                new HashSet<>(ImmutableSet.of("b")),
-                "{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[\"b\"]}}",
+                new HashSet<>(),
+                "{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[\"a\"]}}",
                 "{\"properties\":{},\"_source\":null}",
                 false
             },
@@ -145,13 +145,14 @@ public class MockEsInstallTest {
                 Mappings.builder()
                         .type(ElasticSearchClient.TYPE)
                         .properties(new HashMap<>(
-                            ImmutableMap.of("b", ImmutableMap.of("type", "keyword"))))
+                            ImmutableMap.of("c", ImmutableMap.of("type", "keyword"))))
                         .source(new Mappings.Source())
                     .build(),
                 new HashSet<>(ImmutableSet.of("a", "b")),
-                null,
-                "{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"}},\"_source\":{\"excludes\":[]}}",
-                "{\"properties\":{},\"_source\":null}",
+                new HashSet<>(ImmutableSet.of("c")),
+                "{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"},\"c\":{\"type\":\"keyword\"}}," +
+                    "\"_source\":{\"excludes\":[\"a\",\"b\",\"c\"]}}",
+                "{\"properties\":{\"c\":{\"type\":\"keyword\"}},\"_source\":null}",
                 false
             },
             {
@@ -224,8 +225,8 @@ public class MockEsInstallTest {
                     .build(),
                 new HashSet<>(ImmutableSet.of("a")),
                 new HashSet<>(ImmutableSet.of("b")),
-                "{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"},\"c\":{\"type\":\"keyword\",\"index\":false}},\"" +
-                    "_source\":{\"excludes\":[\"b\"]}}",
+                "{\"properties\":{\"a\":{\"type\":\"keyword\"},\"b\":{\"type\":\"keyword\"},\"c\":{\"type\":\"keyword\",\"index\":false}}," +
+                    "\"_source\":{\"excludes\":[\"a\",\"b\"]}}",
                 "{\"properties\":{\"c\":{\"type\":\"keyword\",\"index\":false}},\"_source\":null}",
                 false
             }
@@ -257,7 +258,7 @@ public class MockEsInstallTest {
         Mappings mappings = structures.getMapping(this.name);
         Assert.assertEquals(this.combineResult, mapper.writeValueAsString(mappings));
 
-        //diff the hisMapping and new, if has new item will update current index
+        //diff the hisMapping and new, if it has new item will update current index
         structures.putStructure(this.name, this.newMappings, new HashMap<>());
         Mappings diff = structures.diffMappings(this.name, hisMappingsClone);
         Assert.assertEquals(this.diffResult, mapper.writeValueAsString(diff));