[Critical] Remove physical index existing check and fix TTL activation bug (#9632)

* Remove physical index existing check

* Make sure instance list ordered in TTL processor to avoid TTL timer never runs.

docs/en/changes/changes.md

@@ -2,7 +2,6 @@
 
 #### Project
 
-
 #### OAP Server
 
 * Add component ID(133) for impala JDBC Java agent plugin and component ID(134) for impala server.
@@ -13,14 +12,19 @@
 * [**Breaking Change**] Change the LAL script format(Add layer property).
 * Adapt ElasticSearch 8.1+, migrate from removed APIs to recommended APIs.
 * Support monitoring MySQL slow SQLs.
+* Remove physical index existing check and keep template existing check only to avoid meaningless `retry wait`
+  in `no-init` mode.
+* Make sure instance list ordered in TTL processor to avoid TTL timer never runs.
 
 #### UI
 
 * Fix: tab active incorrectly, when click tab space
 * Add impala icon for impala JDBC Java agent plugin.
 * (Webapp)Bump up snakeyaml to 1.31 for fixing CVE-2022-25857
-* [Breaking Change]: migrate from Spring Web to Armeria, now you should use the environment variable name `SW_OAP_ADDRESS`
-  to change the OAP backend service addresses, like `SW_OAP_ADDRESS=localhost:12800,localhost:12801`, and use environment
+* [Breaking Change]: migrate from Spring Web to Armeria, now you should use the environment variable
+  name `SW_OAP_ADDRESS`
+  to change the OAP backend service addresses, like `SW_OAP_ADDRESS=localhost:12800,localhost:12801`, and use
+  environment
   variable `SW_SERVER_PORT` to change the port. Other Spring-related configurations don't take effect anymore.
 * Polish the endpoint list graph.
 * Fix styles for an adaptive height.
@@ -31,5 +35,4 @@
 
 #### Documentation
 
-
 All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/149?closed=1)

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/DataTTLKeeperTimer.java

@@ -19,6 +19,7 @@
 package org.apache.skywalking.oap.server.core.storage.ttl;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -76,6 +77,8 @@ public enum DataTTLKeeperTimer {
         List<Model> models = modelGetter.allModels();
 
         List<RemoteInstance> remoteInstances = clusterNodesQuery.queryRemoteNodes();
+        // Sort the instances as same as RemoteClientManager#refresh did.
+        Collections.sort(remoteInstances);
         if (CollectionUtils.isNotEmpty(remoteInstances) && !remoteInstances.get(0).getAddress().isSelf()) {
             log.info(
                 "The selected first getAddress is {}. The remove stage is skipped.",

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java

@@ -44,7 +44,6 @@ public class StorageEsInstaller extends ModelInstaller {
     private final StorageModuleElasticsearchConfig config;
     protected final ColumnTypeEsMapping columnTypeEsMapping;
 
-
     /**
      * The mappings of the template .
      */
@@ -79,16 +78,16 @@ public class StorageEsInstaller extends ModelInstaller {
             }
             return exist;
         }
+
         boolean templateExists = esClient.isExistsTemplate(tableName);
         final Optional<IndexTemplate> template = esClient.getTemplate(tableName);
-        boolean lastIndexExists = esClient.isExistsIndex(TimeSeriesUtils.latestWriteIndexName(model));
 
         if ((templateExists && !template.isPresent()) || (!templateExists && template.isPresent())) {
             throw new Error("[Bug warning] ElasticSearch client query template result is not consistent. " +
                                 "Please file an issue to Apache SkyWalking.(https://github.com/apache/skywalking/issues)");
         }
 
-        boolean exist = templateExists && lastIndexExists;
+        boolean exist = templateExists;
 
         if (exist) {
             structures.putStructure(
@@ -158,13 +157,13 @@ public class StorageEsInstaller extends ModelInstaller {
 
             if (esClient.isExistsIndex(indexName)) {
                 Mappings historyMapping = esClient.getIndex(indexName)
-                        .map(Index::getMappings)
-                        .orElseGet(Mappings::new);
+                                                  .map(Index::getMappings)
+                                                  .orElseGet(Mappings::new);
                 Mappings appendMapping = structures.diffStructure(tableName, historyMapping);
                 if (appendMapping.getProperties() != null && !appendMapping.getProperties().isEmpty()) {
                     boolean isAcknowledged = esClient.updateIndexMapping(indexName, appendMapping);
                     log.info("update {} index finished, isAcknowledged: {}, append mappings: {}", indexName,
-                            isAcknowledged, appendMapping
+                             isAcknowledged, appendMapping
                     );
                     if (!isAcknowledged) {
                         throw new StorageException("update " + indexName + " time series index failure");