[Optimize]优化ES索引及模版的初始化文档(#832)

1、订正不同地方索引模版的shard数存在不一致的问题； 2、删除多余的template.sh，统一使用init_es_template.sh； 3、init_es_template.sh中，增加connect相关索引模版的初始化脚本，删除replica 和 zookeper索引模版的初始化脚本；

[Optimize]优化ES索引及模版的初始化文档(#832)
1、订正不同地方索引模版的shard数存在不一致的问题； 2、删除多余的template.sh，统一使用init_es_template.sh； 3、init_es_template.sh中，增加connect相关索引模版的初始化脚本，删除replica 和 zookeper索引模版的初始化脚本；
ac7c32ac · zengqiao · EricZeng · f4a219ce · ac7c32ac · ac7c32ac
11 changed file
--- a/bin/init_es_template.sh
+++ b/bin/init_es_template.sh
@@ -13,7 +13,7 @@ curl -s --connect-timeout 10 -o /dev/null -X POST -H 'cache-control: no-cache' -
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "2"
      }
    },
    "mappings" : {
@@ -115,7 +115,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "2"
      }
    },
    "mappings" : {
@@ -302,7 +302,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {
@@ -377,7 +377,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {
@@ -436,72 +436,6 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
    "aliases" : { }
  }' 

-curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_replication_metric -d '{
-    "order" : 10,
-    "index_patterns" : [
-      "ks_kafka_replication_metric*"
-    ],
-    "settings" : {
-      "index" : {
-        "number_of_shards" : "10"
-      }
-    },
-    "mappings" : {
-      "properties" : {
-        "brokerId" : {
-          "type" : "long"
-        },
-        "partitionId" : {
-          "type" : "long"
-        },
-        "routingValue" : {
-          "type" : "text",
-          "fields" : {
-            "keyword" : {
-              "ignore_above" : 256,
-              "type" : "keyword"
-            }
-          }
-        },
-        "clusterPhyId" : {
-          "type" : "long"
-        },
-        "topic" : {
-          "type" : "keyword"
-        },
-        "metrics" : {
-          "properties" : {
-            "LogStartOffset" : {
-              "type" : "float"
-            },
-            "Messages" : {
-              "type" : "float"
-            },
-            "LogEndOffset" : {
-              "type" : "float"
-            }
-          }
-        },
-        "key" : {
-          "type" : "text",
-          "fields" : {
-            "keyword" : {
-              "ignore_above" : 256,
-              "type" : "keyword"
-            }
-          }
-        },
-        "timestamp" : {
-          "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
-          "index" : true,
-          "type" : "date",
-          "doc_values" : true
-        }
-      }
-    },
-    "aliases" : { }
-  }'
-
 curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${esaddr}:${port}/_template/ks_kafka_topic_metric -d '{
    "order" : 10,
    "index_patterns" : [
@@ -509,7 +443,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {
@@ -626,7 +560,7 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "2"
      }
    },
    "mappings" : {
@@ -704,6 +638,288 @@ curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: appl
    "aliases" : { }
  }'

+curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_connect_cluster_metric -d '{
+    "order" : 10,
+    "index_patterns" : [
+      "ks_kafka_connect_cluster_metric*"
+    ],
+    "settings" : {
+      "index" : {
+        "number_of_shards" : "2"
+      }
+    },
+    "mappings" : {
+      "properties" : {
+        "connectClusterId" : {
+          "type" : "long"
+        },
+        "routingValue" : {
+          "type" : "text",
+          "fields" : {
+            "keyword" : {
+              "ignore_above" : 256,
+              "type" : "keyword"
+            }
+          }
+        },
+        "clusterPhyId" : {
+          "type" : "long"
+        },
+        "metrics" : {
+          "properties" : {
+            "ConnectorCount" : {
+              "type" : "float"
+            },
+            "TaskCount" : {
+              "type" : "float"
+            },
+            "ConnectorStartupAttemptsTotal" : {
+              "type" : "float"
+            },
+            "ConnectorStartupFailurePercentage" : {
+              "type" : "float"
+            },
+            "ConnectorStartupFailureTotal" : {
+              "type" : "float"
+            },
+            "ConnectorStartupSuccessPercentage" : {
+              "type" : "float"
+            },
+            "ConnectorStartupSuccessTotal" : {
+              "type" : "float"
+            },
+            "TaskStartupAttemptsTotal" : {
+              "type" : "float"
+            },
+            "TaskStartupFailurePercentage" : {
+              "type" : "float"
+            },
+            "TaskStartupFailureTotal" : {
+              "type" : "float"
+            },
+            "TaskStartupSuccessPercentage" : {
+              "type" : "float"
+            },
+            "TaskStartupSuccessTotal" : {
+              "type" : "float"
+            }
+          }
+        },
+        "key" : {
+          "type" : "text",
+          "fields" : {
+            "keyword" : {
+              "ignore_above" : 256,
+              "type" : "keyword"
+            }
+          }
+        },
+        "timestamp" : {
+          "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
+          "index" : true,
+          "type" : "date",
+          "doc_values" : true
+        }
+      }
+    },
+    "aliases" : { }
+  }'
+
+curl -s -o /dev/null -X POST -H 'cache-control: no-cache' -H 'content-type: application/json' http://${SERVER_ES_ADDRESS}/_template/ks_kafka_connect_connector_metric -d '{
+    "order" : 10,
+    "index_patterns" : [
+      "ks_kafka_connect_connector_metric*"
+    ],
+    "settings" : {
+      "index" : {
+        "number_of_shards" : "2"
+      }
+    },
+    "mappings" : {
+      "properties" : {
+        "connectClusterId" : {
+          "type" : "long"
+        },
+        "routingValue" : {
+          "type" : "text",
+          "fields" : {
+            "keyword" : {
+              "ignore_above" : 256,
+              "type" : "keyword"
+            }
+          }
+        },
+        "connectorName" : {
+          "type" : "keyword"
+        },
+        "connectorNameAndClusterId" : {
+          "type" : "keyword"
+        },
+        "clusterPhyId" : {
+          "type" : "long"
+        },
+        "metrics" : {
+          "properties" : {
+            "HealthState" : {
+              "type" : "float"
+            },
+            "ConnectorTotalTaskCount" : {
+              "type" : "float"
+            },
+            "HealthCheckPassed" : {
+              "type" : "float"
+            },
+            "HealthCheckTotal" : {
+              "type" : "float"
+            },
+            "ConnectorRunningTaskCount" : {
+              "type" : "float"
+            },
+            "ConnectorPausedTaskCount" : {
+              "type" : "float"
+            },
+            "ConnectorFailedTaskCount" : {
+              "type" : "float"
+            },
+            "ConnectorUnassignedTaskCount" : {
+              "type" : "float"
+            },
+            "BatchSizeAvg" : {
+              "type" : "float"
+            },
+            "BatchSizeMax" : {
+              "type" : "float"
+            },
+            "OffsetCommitAvgTimeMs" : {
+              "type" : "float"
+            },
+            "OffsetCommitMaxTimeMs" : {
+              "type" : "float"
+            },
+             "OffsetCommitFailurePercentage" : {
+               "type" : "float"
+             },
+             "OffsetCommitSuccessPercentage" : {
+               "type" : "float"
+             },
+             "PollBatchAvgTimeMs" : {
+               "type" : "float"
+             },
+             "PollBatchMaxTimeMs" : {
+               "type" : "float"
+             },
+             "SourceRecordActiveCount" : {
+               "type" : "float"
+             },
+             "SourceRecordActiveCountAvg" : {
+               "type" : "float"
+             },
+             "SourceRecordActiveCountMax" : {
+               "type" : "float"
+             },
+             "SourceRecordPollRate" : {
+               "type" : "float"
+             },
+             "SourceRecordPollTotal" : {
+               "type" : "float"
+             },
+             "SourceRecordWriteRate" : {
+               "type" : "float"
+             },
+             "SourceRecordWriteTotal" : {
+               "type" : "float"
+             },
+           "OffsetCommitCompletionRate" : {
+             "type" : "float"
+           },
+           "OffsetCommitCompletionTotal" : {
+             "type" : "float"
+           },
+           "OffsetCommitSkipRate" : {
+             "type" : "float"
+           },
+           "OffsetCommitSkipTotal" : {
+             "type" : "float"
+           },
+           "PartitionCount" : {
+             "type" : "float"
+           },
+           "PutBatchAvgTimeMs" : {
+             "type" : "float"
+           },
+           "PutBatchMaxTimeMs" : {
+             "type" : "float"
+           },
+           "SinkRecordActiveCount" : {
+             "type" : "float"
+           },
+           "SinkRecordActiveCountAvg" : {
+             "type" : "float"
+           },
+           "SinkRecordActiveCountMax" : {
+             "type" : "float"
+           },
+           "SinkRecordLagMax" : {
+             "type" : "float"
+           },
+           "SinkRecordReadRate" : {
+             "type" : "float"
+           },
+           "SinkRecordReadTotal" : {
+             "type" : "float"
+           },
+           "SinkRecordSendRate" : {
+             "type" : "float"
+           },
+           "SinkRecordSendTotal" : {
+             "type" : "float"
+           },
+           "DeadletterqueueProduceFailures" : {
+             "type" : "float"
+           },
+           "DeadletterqueueProduceRequests" : {
+             "type" : "float"
+           },
+           "LastErrorTimestamp" : {
+             "type" : "float"
+           },
+           "TotalErrorsLogged" : {
+             "type" : "float"
+           },
+           "TotalRecordErrors" : {
+             "type" : "float"
+           },
+           "TotalRecordFailures" : {
+             "type" : "float"
+           },
+           "TotalRecordsSkipped" : {
+             "type" : "float"
+           },
+           "TotalRetries" : {
+             "type" : "float"
+           }
+          }
+        },
+        "key" : {
+          "type" : "text",
+          "fields" : {
+            "keyword" : {
+              "ignore_above" : 256,
+              "type" : "keyword"
+            }
+          }
+        },
+        "timestamp" : {
+          "format" : "yyyy-MM-dd HH:mm:ss Z||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd HH:mm:ss.SSS Z||yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss,SSS||yyyy/MM/dd HH:mm:ss||yyyy-MM-dd HH:mm:ss,SSS Z||yyyy/MM/dd HH:mm:ss,SSS Z||epoch_millis",
+          "index" : true,
+          "type" : "date",
+          "doc_values" : true
+        }
+      }
+    },
+    "aliases" : { }
+  }'
+
 for i in {0..6};
 do
    logdate=_$(date -d "${i} day ago" +%Y-%m-%d)
@@ -711,8 +927,9 @@ do
    curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_cluster_metric${logdate} && \
    curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_group_metric${logdate} && \
    curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_partition_metric${logdate} && \
-    curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_replication_metric${logdate} && \
    curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_zookeeper_metric${logdate} && \
+    curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_connect_cluster_metric${logdate} && \
+    curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_connect_connector_metric${logdate} && \
    curl -s -o /dev/null -X PUT http://${esaddr}:${port}/ks_kafka_topic_metric${logdate} || \
    exit 2
 done
--- a/docs/dev_guide/无数据排查文档.md
+++ b/docs/dev_guide/无数据排查文档.md
@@ -216,7 +216,7 @@ curl http://{ES的IP地址}:{ES的端口号}/_cat/indices/ks_kafka*  查看KS索

 #### 3.1.2、解决方案

-执行[/km-dist/init/template/template.sh](https://github.com/didi/KnowStreaming/blob/master/km-dist/init/template/template.sh)脚本创建索引。
+执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来创建索引及模版。



@@ -245,8 +245,7 @@ curl -XDELETE {ES的IP地址}:{ES的端口号}/ks_kafka*
 curl -XDELETE {ES的IP地址}:{ES的端口号}/_template/ks_kafka*
 ```

-执行[/km-dist/init/template/template.sh](https://github.com/didi/KnowStreaming/blob/master/km-dist/init/template/template.sh)脚本初始化索引和模板。
-
+执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来创建索引及模版。


 ### 3.3、原因三：集群Shard满
@@ -283,4 +282,4 @@ curl -XPUT  -H"content-type:application/json" http://{ES的IP地址}:{ES的端
 }'
 ```

-执行[/km-dist/init/template/template.sh](https://github.com/didi/KnowStreaming/blob/master/km-dist/init/template/template.sh)脚本补全索引。
+执行 [ES索引及模版初始化](https://github.com/didi/KnowStreaming/blob/master/bin/init_es_template.sh) 脚本，来补全索引。
--- a/km-dist/init/template/ks_kafka_group_metric
+++ b/km-dist/init/template/ks_kafka_group_metric
@@ -5,7 +5,7 @@
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {

--- a/km-dist/init/template/ks_kafka_partition_metric
+++ b/km-dist/init/template/ks_kafka_partition_metric
@@ -5,7 +5,7 @@
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {

--- a/km-dist/init/template/ks_kafka_topic_metric
+++ b/km-dist/init/template/ks_kafka_topic_metric
@@ -5,7 +5,7 @@
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {

--- a/km-dist/init/template/ks_kafka_zookeeper_metric
+++ b/km-dist/init/template/ks_kafka_zookeeper_metric
@@ -5,7 +5,7 @@
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "2"
      }
    },
    "mappings" : {

--- a/km-dist/init/template/template.sh
+++ b/km-dist/init/template/template.sh
--- a/km-persistence/src/main/resources/es/template/ks_kafka_group_metric
+++ b/km-persistence/src/main/resources/es/template/ks_kafka_group_metric
@@ -5,7 +5,7 @@
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {

--- a/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric
+++ b/km-persistence/src/main/resources/es/template/ks_kafka_partition_metric
@@ -5,7 +5,7 @@
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {

--- a/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric
+++ b/km-persistence/src/main/resources/es/template/ks_kafka_topic_metric
@@ -5,7 +5,7 @@
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "6"
      }
    },
    "mappings" : {

--- a/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric
+++ b/km-persistence/src/main/resources/es/template/ks_kafka_zookeeper_metric
@@ -5,7 +5,7 @@
    ],
    "settings" : {
      "index" : {
-        "number_of_shards" : "10"
+        "number_of_shards" : "2"
      }
    },
    "mappings" : {