提交 f7ca2672 编写于 作者: L Lemon 提交者: wu-sheng

Database metrics provided. (#2025)

* Add Database source

* Add DatabaseSource source and database_* script for oal

* Enable Database_Indicator

* Add file license

* Rename DatabaseService to ClientDatabase. In the future, we may have real database server monitoring.

* Add query database from H2

* refactor

* update submodule

* refactor

* Fix style

* Fix: unknown tag error

* fix checkstyle
上级 a36394d9
......@@ -37,6 +37,7 @@ SRC_SERVICE_INSTANCE_JVM_CPU: 'ServiceInstanceJVMCPU';
SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory';
SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool';
SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC';
SRC_DATABASE_ACCESS: 'DatabaseAccess';
// Literals
......
......@@ -46,7 +46,7 @@ filterExpression
;
source
: SRC_ALL | SRC_SERVICE | SRC_SERVICE_INSTANCE | SRC_ENDPOINT |
: SRC_ALL | SRC_SERVICE | SRC_DATABASE_ACCESS | SRC_SERVICE_INSTANCE | SRC_ENDPOINT |
SRC_SERVICE_RELATION | SRC_SERVICE_INSTANCE_RELATION | SRC_ENDPOINT_RELATION |
SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC// JVM source of service instance
;
......
......@@ -36,4 +36,5 @@ public class DispatcherContext {
private List<AnalysisResult> serviceInstanceJVMMemoryIndicators = new LinkedList<>();
private List<AnalysisResult> serviceInstanceJVMMemoryPoolIndicators = new LinkedList<>();
private List<AnalysisResult> serviceInstanceJVMGCIndicators = new LinkedList<>();
private List<AnalysisResult> databaseAccessIndicators = new LinkedList<>();
}
......@@ -51,6 +51,10 @@ public class FileGenerator {
createFile(file);
this.generateServiceDispatcher(new FileWriter(file));
file = new File(outputPath, "generated/databaseaccess/DatabaseAccessDispatcher.java");
createFile(file);
this.generateDatabaseAccessDispatcher(new FileWriter(file));
file = new File(outputPath, "generated/servicerelation/ServiceRelationDispatcher.java");
createFile(file);
this.generateServiceRelationDispatcher(new FileWriter(file));
......@@ -127,6 +131,10 @@ public class FileGenerator {
configuration.getTemplate("ServiceDispatcherTemplate.ftl").process(dispatcherContext, output);
}
void generateDatabaseAccessDispatcher(Writer output) throws IOException, TemplateException {
configuration.getTemplate("DatabaseAccessDispatcherTemplate.ftl").process(dispatcherContext, output);
}
void generateServiceRelationDispatcher(Writer output) throws IOException, TemplateException {
configuration.getTemplate("ServiceRelationDispatcherTemplate.ftl").process(dispatcherContext, output);
}
......@@ -201,6 +209,9 @@ public class FileGenerator {
case "ServiceInstanceJVMGC":
dispatcherContext.getServiceInstanceJVMGCIndicators().add(result);
break;
case "DatabaseAccess":
dispatcherContext.getDatabaseAccessIndicators().add(result);
break;
default:
throw new RuntimeException("Unexpected dispatcher");
}
......
......@@ -90,6 +90,12 @@ public class SourceColumnsFactory {
SourceColumn destServiceInstance = new SourceColumn("childServiceInstanceId", "child_service_instance_id", int.class, false);
columnList.add(destServiceInstance);
return columnList;
case "DatabaseAccess":
columnList = new LinkedList<>();
// Service id;
idColumn = new SourceColumn("entityId", "entity_id", String.class, true);
columnList.add(idColumn);
return columnList;
default:
throw new IllegalArgumentException("Illegal source :" + source);
}
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.analysis.generated.databaseaccess;
import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
<#if (databaseAccessIndicators?size>0)>
import org.apache.skywalking.oap.server.core.analysis.worker.IndicatorProcess;
<#list databaseAccessIndicators as indicator>
<#if indicator.filterExpressions??>
import org.apache.skywalking.oap.server.core.analysis.indicator.expression.*;
<#break>
</#if>
</#list>
</#if>
import org.apache.skywalking.oap.server.core.source.*;
/**
* This class is auto generated. Please don't change this class manually.
*
* @author Observability Analysis Language code generator
*/
public class DatabaseAccessDispatcher implements SourceDispatcher<DatabaseAccess> {
@Override public void dispatch(DatabaseAccess source) {
<#list databaseAccessIndicators as indicator>
do${indicator.metricName}(source);
</#list>
}
<#list databaseAccessIndicators as indicator>
private void do${indicator.metricName}(DatabaseAccess source) {
${indicator.metricName}Indicator indicator = new ${indicator.metricName}Indicator();
<#if indicator.filterExpressions??>
<#list indicator.filterExpressions as filterExpression>
if (!new ${filterExpression.expressionObject}().setLeft(${filterExpression.left}).setRight(${filterExpression.right}).match()) {
return;
}
</#list>
</#if>
indicator.setTimeBucket(source.getTimeBucket());
<#list indicator.fieldsFromSource as field>
indicator.${field.fieldSetter}(source.${field.fieldGetter}());
</#list>
indicator.${indicator.entryMethod.methodName}(<#list indicator.entryMethod.argsExpressions as arg>${arg}<#if arg_has_next>, </#if></#list>);
IndicatorProcess.INSTANCE.in(indicator);
}
</#list>
}
......@@ -70,4 +70,13 @@ instance_jvm_memory_noheap_max = from(ServiceInstanceJVMMemory.max).filter(heapS
instance_jvm_young_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.NEW).longAvg();
instance_jvm_old_gc_time = from(ServiceInstanceJVMGC.time).filter(phrase == GCPhrase.OLD).longAvg();
instance_jvm_young_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.NEW).sum();
instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum();
\ No newline at end of file
instance_jvm_old_gc_count = from(ServiceInstanceJVMGC.count).filter(phrase == GCPhrase.OLD).sum();
database_access_resp_time = from(DatabaseAccess.latency).longAvg();
database_access_sla = from(DatabaseAccess.*).percent(status == true);
database_access_cpm = from(DatabaseAccess.*).cpm();
database_access_p99 = from(DatabaseAccess.latency).p99(10);
database_access_p95 = from(DatabaseAccess.latency).p95(10);
database_access_p90 = from(DatabaseAccess.latency).p90(10);
database_access_p75 = from(DatabaseAccess.latency).p75(10);
database_access_p50 = from(DatabaseAccess.latency).p50(10);
\ No newline at end of file
......@@ -77,6 +77,10 @@ public class MetadataQueryService implements org.apache.skywalking.oap.server.li
return getMetadataQueryDAO().getAllServices(startTimestamp, endTimestamp);
}
public List<Database> getAllDatabases() throws IOException {
return getMetadataQueryDAO().getAllDatabases();
}
public List<Service> searchServices(final long startTimestamp, final long endTimestamp,
final String keyword) throws IOException {
return getMetadataQueryDAO().searchServices(startTimestamp, endTimestamp, keyword);
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.query.entity;
import lombok.Getter;
import lombok.Setter;
/**
* @author: liuhaoyang
**/
@Getter
@Setter
public class Database {
private int id;
private String name;
private String type;
}
......@@ -195,6 +195,11 @@ public class ServiceInventory extends RegisterSource {
return true;
}
public static class PropertyUtil {
public static final String DATABASE = "database";
}
public static class Builder implements StorageBuilder<ServiceInventory> {
@Override public ServiceInventory map2Data(Map<String, Object> dbMap) {
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.source;
import lombok.Getter;
import lombok.Setter;
/**
* @author: liuhaoyang
*/
public class DatabaseAccess extends Source {
@Override
public Scope scope() {
return Scope.DatabaseAccess;
}
@Override
public String getEntityId() {
return String.valueOf(id);
}
@Getter @Setter private long id;
@Getter @Setter private String name;
@Getter @Setter private int databaseTypeId;
@Getter @Setter private int latency;
@Getter @Setter private boolean status;
}
......@@ -24,7 +24,7 @@ package org.apache.skywalking.oap.server.core.source;
public enum Scope {
All, Service, ServiceInstance, Endpoint, ServiceRelation, ServiceInstanceRelation, EndpointRelation, NetworkAddress,
ServiceInstanceJVMCPU, ServiceInstanceJVMMemory, ServiceInstanceJVMMemoryPool, ServiceInstanceJVMGC,
Segment, Alarm, ServiceInventory, ServiceInstanceInventory, EndpointInventory;
Segment, Alarm, ServiceInventory, ServiceInstanceInventory, EndpointInventory, DatabaseAccess;
public static Scope valueOf(int ordinal) {
if (ordinal < 0 || ordinal >= values().length) {
......
......@@ -36,6 +36,8 @@ public interface IMetadataQueryDAO extends DAO {
List<Service> getAllServices(final long startTimestamp, final long endTimestamp) throws IOException;
List<Database> getAllDatabases() throws IOException;
List<Service> searchServices(final long startTimestamp, final long endTimestamp,
final String keyword) throws IOException;
......
......@@ -89,4 +89,8 @@ public class MetadataQuery implements GraphQLQueryResolver {
public EndpointInfo getEndpointInfo(final int endpointId) throws IOException {
return getMetadataQueryService().getEndpointInfo(endpointId);
}
public List<Database> getAllDatabases(final Duration duration) throws IOException {
return getMetadataQueryService().getAllDatabases();
}
}
Subproject commit c65a23bd6b9bba8d1df30d4de261624952df2b7b
Subproject commit 85b81e2e34efb0b670d039154feca336c9203700
......@@ -211,12 +211,16 @@ public class MultiScopesSpanListener implements EntrySpanListener, ExitSpanListe
exitSourceBuilder.setTimeBucket(minuteTimeBucket);
sourceReceiver.receive(exitSourceBuilder.toServiceRelation());
sourceReceiver.receive(exitSourceBuilder.toServiceInstanceRelation());
if (RequestType.DATABASE.equals(exitSourceBuilder.getType())) {
sourceReceiver.receive(exitSourceBuilder.toDatabaseAccess());
}
});
}
public static class Factory implements SpanListenerFactory {
@Override public SpanListener create(ModuleManager moduleManager) {
@Override
public SpanListener create(ModuleManager moduleManager) {
return new MultiScopesSpanListener(moduleManager);
}
}
......
......@@ -171,4 +171,18 @@ class SourceBuilder {
endpointRelation.setTimeBucket(timeBucket);
return endpointRelation;
}
DatabaseAccess toDatabaseAccess() {
if (!RequestType.DATABASE.equals(type)) {
return null;
}
DatabaseAccess databaseAccess = new DatabaseAccess();
databaseAccess.setId(destServiceId);
databaseAccess.setDatabaseTypeId(componentId);
databaseAccess.setLatency(latency);
databaseAccess.setName(destServiceName);
databaseAccess.setStatus(status);
databaseAccess.setTimeBucket(timeBucket);
return databaseAccess;
}
}
......@@ -20,8 +20,10 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query;
import com.google.common.base.Strings;
import com.google.gson.*;
import java.io.IOException;
import java.util.*;
import org.apache.skywalking.oap.server.core.query.entity.*;
import org.apache.skywalking.oap.server.core.register.*;
import org.apache.skywalking.oap.server.core.source.DetectPoint;
......@@ -105,6 +107,38 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
return buildServices(response);
}
@Override
public List<Database> getAllDatabases() throws IOException {
SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Database.value()));
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.size(100);
SearchResponse response = getClient().search(ServiceInventory.MODEL_NAME, sourceBuilder);
List<Database> databases = new ArrayList<>();
for (SearchHit searchHit : response.getHits()) {
Map<String, Object> sourceAsMap = searchHit.getSourceAsMap();
Database database = new Database();
database.setId(((Number) sourceAsMap.get(ServiceInventory.SEQUENCE)).intValue());
database.setName((String) sourceAsMap.get(ServiceInventory.NAME));
String propertiesString = (String) sourceAsMap.get(ServiceInstanceInventory.PROPERTIES);
if (!Strings.isNullOrEmpty(propertiesString)) {
JsonObject properties = GSON.fromJson(propertiesString, JsonObject.class);
if (properties.has(ServiceInventory.PropertyUtil.DATABASE)) {
database.setType(properties.get(ServiceInventory.PropertyUtil.DATABASE).getAsString());
} else {
database.setType("UNKNOWN");
}
}
databases.add(database);
}
return databases;
}
@Override public List<Service> searchServices(long startTimestamp, long endTimestamp,
String keyword) throws IOException {
SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
......
......@@ -20,9 +20,11 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao;
import com.google.common.base.Strings;
import com.google.gson.*;
import java.io.IOException;
import java.sql.*;
import java.util.*;
import org.apache.skywalking.oap.server.core.query.entity.*;
import org.apache.skywalking.oap.server.core.register.*;
import org.apache.skywalking.oap.server.core.source.DetectPoint;
......@@ -44,7 +46,8 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
this.h2Client = h2Client;
}
@Override public int numOfService(long startTimestamp, long endTimestamp) throws IOException {
@Override
public int numOfService(long startTimestamp, long endTimestamp) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select count(*) num from ").append(ServiceInventory.MODEL_NAME).append(" where ");
......@@ -63,7 +66,8 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
return 0;
}
@Override public int numOfEndpoint(long startTimestamp, long endTimestamp) throws IOException {
@Override
public int numOfEndpoint(long startTimestamp, long endTimestamp) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select count(*) num from ").append(EndpointInventory.MODEL_NAME).append(" where ");
......@@ -82,8 +86,9 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
return 0;
}
@Override public int numOfConjectural(long startTimestamp, long endTimestamp,
int nodeTypeValue) throws IOException {
@Override
public int numOfConjectural(long startTimestamp, long endTimestamp,
int nodeTypeValue) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select count(*) num from ").append(ServiceInventory.MODEL_NAME).append(" where ");
......@@ -120,8 +125,42 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
}
}
@Override public List<Service> searchServices(long startTimestamp, long endTimestamp,
String keyword) throws IOException {
@Override
public List<Database> getAllDatabases() throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(1);
sql.append("select * from ").append(ServiceInventory.MODEL_NAME).append(" where ");
sql.append(ServiceInventory.NODE_TYPE).append("=? limit 100");
condition.add(NodeType.Database.value());
try (Connection connection = h2Client.getConnection()) {
try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0]))) {
List<Database> databases = new ArrayList<>();
while (resultSet.next()) {
Database database = new Database();
database.setId(resultSet.getInt(ServiceInventory.SEQUENCE));
database.setName(resultSet.getString(ServiceInventory.NAME));
String propertiesString = resultSet.getString(ServiceInstanceInventory.PROPERTIES);
if (!Strings.isNullOrEmpty(propertiesString)) {
JsonObject properties = GSON.fromJson(propertiesString, JsonObject.class);
if (properties.has(ServiceInventory.PropertyUtil.DATABASE)) {
database.setType(properties.get(ServiceInventory.PropertyUtil.DATABASE).getAsString());
} else {
database.setType("UNKNOWN");
}
}
databases.add(database);
}
return databases;
}
} catch (SQLException e) {
throw new IOException(e);
}
}
@Override
public List<Service> searchServices(long startTimestamp, long endTimestamp,
String keyword) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(ServiceInventory.MODEL_NAME).append(" where ");
......@@ -142,7 +181,8 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
}
}
@Override public Service searchService(String serviceCode) throws IOException {
@Override
public Service searchService(String serviceCode) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(ServiceInventory.MODEL_NAME).append(" where ");
......@@ -168,8 +208,9 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
return null;
}
@Override public List<Endpoint> searchEndpoint(String keyword, String serviceId,
int limit) throws IOException {
@Override
public List<Endpoint> searchEndpoint(String keyword, String serviceId,
int limit) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(EndpointInventory.MODEL_NAME).append(" where ");
......@@ -199,8 +240,9 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
return endpoints;
}
@Override public List<ServiceInstance> getServiceInstances(long startTimestamp, long endTimestamp,
String serviceId) throws IOException {
@Override
public List<ServiceInstance> getServiceInstances(long startTimestamp, long endTimestamp,
String serviceId) throws IOException {
StringBuilder sql = new StringBuilder();
List<Object> condition = new ArrayList<>(5);
sql.append("select * from ").append(ServiceInstanceInventory.MODEL_NAME).append(" where ");
......@@ -255,13 +297,13 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
}
private void setTimeRangeCondition(StringBuilder sql, List<Object> conditions, long startTimestamp,
long endTimestamp) {
long endTimestamp) {
sql.append(" ( (").append(RegisterSource.HEARTBEAT_TIME).append(" >= ? and ")
.append(RegisterSource.REGISTER_TIME).append(" <= ? )");
.append(RegisterSource.REGISTER_TIME).append(" <= ? )");
conditions.add(endTimestamp);
conditions.add(endTimestamp);
sql.append(" or (").append(RegisterSource.REGISTER_TIME).append(" <= ? and ")
.append(RegisterSource.HEARTBEAT_TIME).append(" >= ? ) ) ");
.append(RegisterSource.HEARTBEAT_TIME).append(" >= ? ) ) ");
conditions.add(endTimestamp);
conditions.add(startTimestamp);
}
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册