From 2ea341b890b3c89bc6420e1a59be2b4aa0c8ec75 Mon Sep 17 00:00:00 2001 From: mrproliu <741550557@qq.com> Date: Tue, 25 Feb 2020 10:12:16 +0800 Subject: [PATCH] Add missed profiled segment query (#4413) * add profiled segment query * fix query field error --- .../core/query/ProfileTaskQueryService.java | 127 ++++++++++++++++++ .../core/query/entity/ProfiledSegment.java | 37 +++++ .../core/query/entity/ProfiledSpan.java | 49 +++++++ .../IProfileThreadSnapshotQueryDAO.java | 6 + .../profile/analyze/ProfileStackAnalyze.java | 6 + .../query/graphql/resolver/ProfileQuery.java | 5 + .../src/main/resources/query-protocol | 2 +- .../ProfileThreadSnapshotQueryEsDAO.java | 31 +++++ .../query/ProfileThreadSnapshotQuery.java | 42 ++++++ .../dao/H2ProfileThreadSnapshotQueryDAO.java | 32 +++++ .../skywalking/e2e/profile/ProfileClient.java | 21 +++ .../e2e/profile/query/ProfiledSegment.java | 36 +++++ .../profile/query/ProfiledSegmentMatcher.java | 45 +++++++ .../e2e/profile/query/ProfiledSpan.java | 40 ++++++ .../profile/query/ProfiledSpanMatcher.java | 43 ++++++ ...ofileVerificationITCase.profileSegment.yml | 47 +++++++ .../src/main/resources/getProfiledSegment.gql | 28 ++++ .../e2e/ProfileVerificationITCase.java | 11 ++ 18 files changed, 607 insertions(+), 1 deletion(-) create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ProfiledSegment.java create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ProfiledSpan.java create mode 100644 test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSegment.java create mode 100644 test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSegmentMatcher.java create mode 100644 test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSpan.java create mode 100644 test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSpanMatcher.java create mode 100644 test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/expected-data/org.apache.skywalking.e2e.ProfileVerificationITCase.profileSegment.yml create mode 100644 test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/getProfiledSegment.gql diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProfileTaskQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProfileTaskQueryService.java index 72012ccb66..7d02ef9e58 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProfileTaskQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProfileTaskQueryService.java @@ -20,18 +20,31 @@ package org.apache.skywalking.oap.server.core.query; import com.google.common.base.Objects; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; + +import org.apache.skywalking.apm.network.language.agent.v2.SegmentObject; +import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.CoreModuleConfig; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.cache.EndpointInventoryCache; +import org.apache.skywalking.oap.server.core.cache.NetworkAddressInventoryCache; import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache; import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache; +import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; import org.apache.skywalking.oap.server.core.profile.analyze.ProfileAnalyzer; import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; +import org.apache.skywalking.oap.server.core.query.entity.KeyValue; +import org.apache.skywalking.oap.server.core.query.entity.LogEntity; import org.apache.skywalking.oap.server.core.query.entity.ProfileAnalyzation; import org.apache.skywalking.oap.server.core.query.entity.ProfileTask; import org.apache.skywalking.oap.server.core.query.entity.ProfileTaskLog; +import org.apache.skywalking.oap.server.core.query.entity.ProfiledSegment; +import org.apache.skywalking.oap.server.core.query.entity.ProfiledSpan; +import org.apache.skywalking.oap.server.core.register.EndpointInventory; import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory; import org.apache.skywalking.oap.server.core.register.ServiceInventory; import org.apache.skywalking.oap.server.core.storage.StorageModule; @@ -43,6 +56,7 @@ import org.apache.skywalking.oap.server.library.module.Service; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; /** * handle profile task queries @@ -54,6 +68,9 @@ public class ProfileTaskQueryService implements Service { private IProfileThreadSnapshotQueryDAO profileThreadSnapshotQueryDAO; private ServiceInventoryCache serviceInventoryCache; private ServiceInstanceInventoryCache serviceInstanceInventoryCache; + private NetworkAddressInventoryCache networkAddressInventoryCache; + private IComponentLibraryCatalogService componentLibraryCatalogService; + private EndpointInventoryCache endpointInventoryCache; private final ProfileAnalyzer profileAnalyzer; @@ -107,6 +124,33 @@ public class ProfileTaskQueryService implements Service { return profileThreadSnapshotQueryDAO; } + private NetworkAddressInventoryCache getNetworkAddressInventoryCache() { + if (networkAddressInventoryCache == null) { + this.networkAddressInventoryCache = moduleManager.find(CoreModule.NAME) + .provider() + .getService(NetworkAddressInventoryCache.class); + } + return networkAddressInventoryCache; + } + + private IComponentLibraryCatalogService getComponentLibraryCatalogService() { + if (componentLibraryCatalogService == null) { + this.componentLibraryCatalogService = moduleManager.find(CoreModule.NAME) + .provider() + .getService(IComponentLibraryCatalogService.class); + } + return componentLibraryCatalogService; + } + + private EndpointInventoryCache getEndpointInventoryCache() { + if (endpointInventoryCache == null) { + this.endpointInventoryCache = moduleManager.find(CoreModule.NAME) + .provider() + .getService(EndpointInventoryCache.class); + } + return endpointInventoryCache; + } + /** * search profile task list * @@ -158,4 +202,87 @@ public class ProfileTaskQueryService implements Service { return profileAnalyzer.analyze(segmentId, start, end); } + public ProfiledSegment getProfiledSegment(String segmentId) throws IOException { + SegmentRecord segmentRecord = getProfileThreadSnapshotQueryDAO().getProfiledSegment(segmentId); + if (segmentRecord == null) { + return null; + } + + ProfiledSegment profiledSegment = new ProfiledSegment(); + SegmentObject segmentObject = SegmentObject.parseFrom(segmentRecord.getDataBinary()); + profiledSegment.getSpans().addAll(buildProfiledSpanList(segmentObject)); + + return profiledSegment; + } + + private List buildProfiledSpanList(SegmentObject segmentObject) { + List spans = new ArrayList<>(); + + segmentObject.getSpansList().forEach(spanObject -> { + ProfiledSpan span = new ProfiledSpan(); + span.setSpanId(spanObject.getSpanId()); + span.setParentSpanId(spanObject.getParentSpanId()); + span.setStartTime(spanObject.getStartTime()); + span.setEndTime(spanObject.getEndTime()); + span.setError(spanObject.getIsError()); + span.setLayer(spanObject.getSpanLayer().name()); + span.setType(spanObject.getSpanType().name()); + + if (spanObject.getPeerId() == 0) { + span.setPeer(spanObject.getPeer()); + } else { + span.setPeer(getNetworkAddressInventoryCache().get(spanObject.getPeerId()).getName()); + } + + String endpointName = spanObject.getOperationName(); + if (spanObject.getOperationNameId() != 0) { + EndpointInventory endpointInventory = getEndpointInventoryCache().get(spanObject.getOperationNameId()); + if (nonNull(endpointInventory)) { + endpointName = endpointInventory.getName(); + } else { + endpointName = Const.EMPTY_STRING; + } + } + span.setEndpointName(endpointName); + + final ServiceInventory serviceInventory = getServiceInventoryCache().get(segmentObject.getServiceId()); + if (serviceInventory != null) { + span.setServiceCode(serviceInventory.getName()); + } else { + span.setServiceCode("unknown"); + } + + if (spanObject.getComponentId() == 0) { + span.setComponent(spanObject.getComponent()); + } else { + span.setComponent(getComponentLibraryCatalogService().getComponentName(spanObject.getComponentId())); + } + + spanObject.getTagsList().forEach(tag -> { + KeyValue keyValue = new KeyValue(); + keyValue.setKey(tag.getKey()); + keyValue.setValue(tag.getValue()); + span.getTags().add(keyValue); + }); + + spanObject.getLogsList().forEach(log -> { + LogEntity logEntity = new LogEntity(); + logEntity.setTime(log.getTime()); + + log.getDataList().forEach(data -> { + KeyValue keyValue = new KeyValue(); + keyValue.setKey(data.getKey()); + keyValue.setValue(data.getValue()); + logEntity.getData().add(keyValue); + }); + + span.getLogs().add(logEntity); + }); + + spans.add(span); + }); + + return spans; + } + } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ProfiledSegment.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ProfiledSegment.java new file mode 100644 index 0000000000..271dcdc450 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ProfiledSegment.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.entity; + +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class ProfiledSegment { + + private final List spans; + + public ProfiledSegment() { + this.spans = new ArrayList<>(); + } + +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ProfiledSpan.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ProfiledSpan.java new file mode 100644 index 0000000000..4e7f521a27 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/entity/ProfiledSpan.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.query.entity; + +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class ProfiledSpan { + + private int spanId; + private int parentSpanId; + private String serviceCode; + private long startTime; + private long endTime; + private String endpointName; + private String type; + private String peer; + private String component; + private boolean isError; + private String layer; + private final List tags; + private final List logs; + + public ProfiledSpan() { + this.tags = new ArrayList<>(); + this.logs = new ArrayList<>(); + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profile/IProfileThreadSnapshotQueryDAO.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profile/IProfileThreadSnapshotQueryDAO.java index dedda807b1..907a418f84 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profile/IProfileThreadSnapshotQueryDAO.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/profile/IProfileThreadSnapshotQueryDAO.java @@ -20,6 +20,8 @@ package org.apache.skywalking.oap.server.core.storage.profile; import java.io.IOException; import java.util.List; + +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord; import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; import org.apache.skywalking.oap.server.core.storage.DAO; @@ -56,4 +58,8 @@ public interface IProfileThreadSnapshotQueryDAO extends DAO { */ List queryRecords(String segmentId, int minSequence, int maxSequence) throws IOException; + /** + * search segment data + */ + SegmentRecord getProfiledSegment(String segmentId) throws IOException; } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileStackAnalyze.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileStackAnalyze.java index 82ecf2bd65..7ceddb37b5 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileStackAnalyze.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profile/analyze/ProfileStackAnalyze.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.stream.Collectors; import lombok.Data; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord; import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; import org.apache.skywalking.oap.server.core.query.entity.ProfileStackTree; @@ -86,6 +87,11 @@ public class ProfileStackAnalyze { .collect(Collectors.toList()); } + @Override + public SegmentRecord getProfiledSegment(String segmentId) throws IOException { + return null; + } + } } diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/ProfileQuery.java b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/ProfileQuery.java index 750f33d647..0f2dff9d96 100644 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/ProfileQuery.java +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/ProfileQuery.java @@ -24,6 +24,7 @@ import org.apache.skywalking.oap.server.core.query.ProfileTaskQueryService; import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; import org.apache.skywalking.oap.server.core.query.entity.ProfileAnalyzation; import org.apache.skywalking.oap.server.core.query.entity.ProfileTask; +import org.apache.skywalking.oap.server.core.query.entity.ProfiledSegment; import org.apache.skywalking.oap.server.library.module.ModuleManager; import java.io.IOException; @@ -58,6 +59,10 @@ public class ProfileQuery implements GraphQLQueryResolver { return getProfileTaskQueryService().getTaskTraces(taskID); } + public ProfiledSegment getProfiledSegment(final String segmentId) throws IOException { + return getProfileTaskQueryService().getProfiledSegment(segmentId); + } + public ProfileAnalyzation getProfileAnalyze(final String segmentId, final long start, final long end) throws IOException { return getProfileTaskQueryService().getProfileAnalyze(segmentId, start, end); diff --git a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol index 1018b79502..6b26ddad20 160000 --- a/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol +++ b/oap-server/server-query-plugin/query-graphql-plugin/src/main/resources/query-protocol @@ -1 +1 @@ -Subproject commit 1018b795021e0d96e6c131c8695fb9ddc9d66916 +Subproject commit 6b26ddad2099b8782b2e298fd0df02dfd1d6609f diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfileThreadSnapshotQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfileThreadSnapshotQueryEsDAO.java index fc352503cf..ce17d57081 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfileThreadSnapshotQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfileThreadSnapshotQueryEsDAO.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; +import com.google.common.base.Strings; import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord; import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; @@ -39,6 +40,7 @@ import org.elasticsearch.search.sort.SortOrder; import java.io.IOException; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -146,6 +148,35 @@ public class ProfileThreadSnapshotQueryEsDAO extends EsDAO implements IProfileTh return result; } + @Override + public SegmentRecord getProfiledSegment(String segmentId) throws IOException { + SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); + sourceBuilder.query(QueryBuilders.termQuery(SegmentRecord.SEGMENT_ID, segmentId)); + sourceBuilder.size(1); + + SearchResponse response = getClient().search(SegmentRecord.INDEX_NAME, sourceBuilder); + + if (response.getHits().getHits().length == 0) { + return null; + } + SearchHit searchHit = response.getHits().getHits()[0]; + SegmentRecord segmentRecord = new SegmentRecord(); + segmentRecord.setSegmentId((String) searchHit.getSourceAsMap().get(SegmentRecord.SEGMENT_ID)); + segmentRecord.setTraceId((String) searchHit.getSourceAsMap().get(SegmentRecord.TRACE_ID)); + segmentRecord.setServiceId(((Number) searchHit.getSourceAsMap().get(SegmentRecord.SERVICE_ID)).intValue()); + segmentRecord.setEndpointName((String) searchHit.getSourceAsMap().get(SegmentRecord.ENDPOINT_NAME)); + segmentRecord.setStartTime(((Number) searchHit.getSourceAsMap().get(SegmentRecord.START_TIME)).longValue()); + segmentRecord.setEndTime(((Number) searchHit.getSourceAsMap().get(SegmentRecord.END_TIME)).longValue()); + segmentRecord.setLatency(((Number) searchHit.getSourceAsMap().get(SegmentRecord.LATENCY)).intValue()); + segmentRecord.setIsError(((Number) searchHit.getSourceAsMap().get(SegmentRecord.IS_ERROR)).intValue()); + String dataBinaryBase64 = (String) searchHit.getSourceAsMap().get(SegmentRecord.DATA_BINARY); + if (!Strings.isNullOrEmpty(dataBinaryBase64)) { + segmentRecord.setDataBinary(Base64.getDecoder().decode(dataBinaryBase64)); + } + segmentRecord.setVersion(((Number) searchHit.getSourceAsMap().get(SegmentRecord.VERSION)).intValue()); + return segmentRecord; + } + protected int querySequenceWithAgg(AbstractAggregationBuilder aggregationBuilder, String segmentId, long start, long end) throws IOException { SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource(); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java index 753ef582e9..83bd9a4fac 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java @@ -33,6 +33,7 @@ import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; import org.apache.skywalking.oap.server.library.util.BooleanUtils; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.elasticsearch.common.Strings; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.WhereQueryImpl; @@ -148,6 +149,47 @@ public class ProfileThreadSnapshotQuery implements IProfileThreadSnapshotQueryDA return result; } + @Override + public SegmentRecord getProfiledSegment(String segmentId) throws IOException { + WhereQueryImpl query = select().column(SegmentRecord.SEGMENT_ID) + .column(SegmentRecord.TRACE_ID) + .column(SegmentRecord.SERVICE_ID) + .column(SegmentRecord.ENDPOINT_NAME) + .column(SegmentRecord.START_TIME) + .column(SegmentRecord.END_TIME) + .column(SegmentRecord.LATENCY) + .column(SegmentRecord.IS_ERROR) + .column(SegmentRecord.DATA_BINARY) + .column(SegmentRecord.VERSION) + .from(client.getDatabase(), SegmentRecord.INDEX_NAME) + .where() + .and(eq(SegmentRecord.SEGMENT_ID, segmentId)); + List series = client.queryForSeries(query); + if (series == null || series.isEmpty()) { + return null; + } + + List values = series.get(0).getValues().get(0); + SegmentRecord segmentRecord = new SegmentRecord(); + + segmentRecord.setSegmentId((String) values.get(1)); + segmentRecord.setTraceId((String) values.get(2)); + segmentRecord.setServiceId((int) values.get(3)); + segmentRecord.setEndpointName((String) values.get(4)); + segmentRecord.setStartTime((long) values.get(5)); + segmentRecord.setEndTime((long) values.get(6)); + segmentRecord.setLatency((int) values.get(7)); + segmentRecord.setIsError((int) values.get(8)); + segmentRecord.setVersion((int) values.get(10)); + + String base64 = (String) values.get(9); + if (!Strings.isNullOrEmpty(base64)) { + segmentRecord.setDataBinary(Base64.getDecoder().decode(base64)); + } + + return segmentRecord; + } + private int querySequenceWithAgg(String function, String segmentId, long start, long end) throws IOException { WhereQueryImpl query = select() .function(function, ProfileThreadSnapshotRecord.SEQUENCE) diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2ProfileThreadSnapshotQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2ProfileThreadSnapshotQueryDAO.java index 53ceb932a2..58214e48cb 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2ProfileThreadSnapshotQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2ProfileThreadSnapshotQueryDAO.java @@ -27,6 +27,8 @@ import java.util.Base64; import java.util.Collections; import java.util.LinkedList; import java.util.List; + +import com.google.common.base.Strings; import org.apache.skywalking.apm.util.StringUtil; import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord; @@ -155,6 +157,36 @@ public class H2ProfileThreadSnapshotQueryDAO implements IProfileThreadSnapshotQu return result; } + @Override + public SegmentRecord getProfiledSegment(String segmentId) throws IOException { + try (Connection connection = h2Client.getConnection()) { + + try (ResultSet resultSet = h2Client.executeQuery(connection, "select * from " + SegmentRecord.INDEX_NAME + " where " + SegmentRecord.SEGMENT_ID + " = ?", segmentId)) { + if (resultSet.next()) { + SegmentRecord segmentRecord = new SegmentRecord(); + segmentRecord.setSegmentId(resultSet.getString(SegmentRecord.SEGMENT_ID)); + segmentRecord.setTraceId(resultSet.getString(SegmentRecord.TRACE_ID)); + segmentRecord.setServiceId(resultSet.getInt(SegmentRecord.SERVICE_ID)); + segmentRecord.setEndpointName(resultSet.getString(SegmentRecord.ENDPOINT_NAME)); + segmentRecord.setStartTime(resultSet.getLong(SegmentRecord.START_TIME)); + segmentRecord.setEndTime(resultSet.getLong(SegmentRecord.END_TIME)); + segmentRecord.setLatency(resultSet.getInt(SegmentRecord.LATENCY)); + segmentRecord.setIsError(resultSet.getInt(SegmentRecord.IS_ERROR)); + String dataBinaryBase64 = resultSet.getString(SegmentRecord.DATA_BINARY); + if (!Strings.isNullOrEmpty(dataBinaryBase64)) { + segmentRecord.setDataBinary(Base64.getDecoder().decode(dataBinaryBase64)); + } + segmentRecord.setVersion(resultSet.getInt(SegmentRecord.VERSION)); + return segmentRecord; + } + } + } catch (SQLException e) { + throw new IOException(e); + } + + return null; + } + private int querySequenceWithAgg(String aggType, String segmentId, long start, long end) throws IOException { StringBuilder sql = new StringBuilder(); sql.append("select ").append(aggType).append("(").append(ProfileThreadSnapshotRecord.SEQUENCE).append(") from ").append(ProfileThreadSnapshotRecord.INDEX_NAME).append(" where "); diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/ProfileClient.java b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/ProfileClient.java index 29261febbf..ce9843eccd 100644 --- a/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/ProfileClient.java +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/ProfileClient.java @@ -121,6 +121,26 @@ public class ProfileClient extends SimpleQueryClient { return Objects.requireNonNull(responseEntity.getBody()).getData().getTraces(); } + public ProfiledSegment.ProfiledSegmentData getProfiledSegment(final String segmentId) throws IOException { + final URL queryFileUrl = Resources.getResource("getProfiledSegment.gql"); + final String queryString = Resources.readLines(queryFileUrl, StandardCharsets.UTF_8) + .stream() + .filter(it -> !it.startsWith("#")) + .collect(Collectors.joining()) + .replace("{segmentId}", segmentId); + final ResponseEntity> responseEntity = restTemplate.exchange( + new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)), + new ParameterizedTypeReference>() { + } + ); + + if (responseEntity.getStatusCode() != HttpStatus.OK) { + throw new RuntimeException("Response status != 200, actual: " + responseEntity.getStatusCode()); + } + + return Objects.requireNonNull(responseEntity.getBody()).getData().getSegment(); + } + public ProfileAnalyzation getProfileAnalyzation(final String segmentId, long start, long end) throws IOException { final URL queryFileUrl = Resources.getResource("getProfileAnalyzation.gql"); final String queryString = Resources.readLines(queryFileUrl, StandardCharsets.UTF_8) @@ -143,4 +163,5 @@ public class ProfileClient extends SimpleQueryClient { return Objects.requireNonNull(responseEntity.getBody()).getData(); } + } diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSegment.java b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSegment.java new file mode 100644 index 0000000000..b36e4a6416 --- /dev/null +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSegment.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.profile.query; + +import lombok.Data; +import lombok.ToString; + +import java.util.List; + +@Data +public class ProfiledSegment { + + private ProfiledSegmentData segment; + + @Data + @ToString + public static class ProfiledSegmentData { + private List spans; + } +} diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSegmentMatcher.java b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSegmentMatcher.java new file mode 100644 index 0000000000..25cf661f6c --- /dev/null +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSegmentMatcher.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.profile.query; + +import com.google.common.primitives.Ints; +import lombok.Data; +import org.apache.skywalking.e2e.verification.AbstractMatcher; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +@Data +public class ProfiledSegmentMatcher extends AbstractMatcher { + + private List spans; + + @Override + public void verify(ProfiledSegment.ProfiledSegmentData profiledSegmentData) { + assertThat(spans).hasSameSizeAs(profiledSegmentData.getSpans()); + + profiledSegmentData.setSpans(profiledSegmentData.getSpans().stream().sorted().collect(Collectors.toList())); + + for (int i = 0; i < profiledSegmentData.getSpans().size(); i++) { + spans.get(i).verify(profiledSegmentData.getSpans().get(i)); + } + } +} diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSpan.java b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSpan.java new file mode 100644 index 0000000000..3d6dd20649 --- /dev/null +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSpan.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.profile.query; + +import com.google.common.primitives.Ints; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class ProfiledSpan implements Comparable { + + private String spanId; + private String parentSpanId; + private String serviceCode; + private String startTime; + private String endTime; + private String endpointName; + + @Override + public int compareTo(ProfiledSpan o) { + return Ints.compare(Integer.parseInt(spanId), Integer.parseInt(o.spanId)); + } +} diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSpanMatcher.java b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSpanMatcher.java new file mode 100644 index 0000000000..93aca7e258 --- /dev/null +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/java/org/apache/skywalking/e2e/profile/query/ProfiledSpanMatcher.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.profile.query; + +import lombok.Data; +import org.apache.skywalking.e2e.verification.AbstractMatcher; + +@Data +public class ProfiledSpanMatcher extends AbstractMatcher { + + private String spanId; + private String parentSpanId; + private String serviceCode; + private String startTime; + private String endTime; + private String endpointName; + + @Override + public void verify(ProfiledSpan span) { + doVerify(spanId, span.getSpanId()); + doVerify(parentSpanId, span.getParentSpanId()); + doVerify(serviceCode, span.getServiceCode()); + doVerify(startTime, span.getStartTime()); + doVerify(endTime, span.getEndTime()); + doVerify(endpointName, span.getEndpointName()); + } +} diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/expected-data/org.apache.skywalking.e2e.ProfileVerificationITCase.profileSegment.yml b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/expected-data/org.apache.skywalking.e2e.ProfileVerificationITCase.profileSegment.yml new file mode 100644 index 0000000000..a7246f2681 --- /dev/null +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/expected-data/org.apache.skywalking.e2e.ProfileVerificationITCase.profileSegment.yml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +spans: + - spanId: 0 + parentSpanId: -1 + serviceCode: not null + startTime: gt 0 + endTime: gt 0 + endpointName: /e2e/users + - spanId: 1 + parentSpanId: 0 + serviceCode: not null + startTime: gt 0 + endTime: gt 0 + endpointName: H2/JDBI/PreparedStatement/executeQuery + - spanId: 2 + parentSpanId: 0 + serviceCode: not null + startTime: gt 0 + endTime: gt 0 + endpointName: H2/JDBI/PreparedStatement/executeUpdate + - spanId: 3 + parentSpanId: 0 + serviceCode: not null + startTime: gt 0 + endTime: gt 0 + endpointName: H2/JDBI/Connection/commit + - spanId: 4 + parentSpanId: 0 + serviceCode: not null + startTime: gt 0 + endTime: gt 0 + endpointName: H2/JDBI/Connection/commit diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/getProfiledSegment.gql b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/getProfiledSegment.gql new file mode 100644 index 0000000000..27a31d8b81 --- /dev/null +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/main/resources/getProfiledSegment.gql @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "query":"query getProfiledSegment($segmentId: String!) { + segment: getProfiledSegment(segmentId: $segmentId) { + spans { + spanId parentSpanId serviceCode startTime endTime endpointName type peer component isError layer + } + } + }", + "variables": { + "segmentId": "{segmentId}" + } +} \ No newline at end of file diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/test/java/org/apache/skywalking/e2e/ProfileVerificationITCase.java b/test/e2e/e2e-profile/e2e-profile-test-runner/src/test/java/org/apache/skywalking/e2e/ProfileVerificationITCase.java index cfba209f0c..5f6ea7fcda 100644 --- a/test/e2e/e2e-profile/e2e-profile-test-runner/src/test/java/org/apache/skywalking/e2e/ProfileVerificationITCase.java +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/test/java/org/apache/skywalking/e2e/ProfileVerificationITCase.java @@ -34,6 +34,8 @@ import org.apache.skywalking.e2e.profile.query.ProfileAnalyzation; import org.apache.skywalking.e2e.profile.query.ProfileStackTreeMatcher; import org.apache.skywalking.e2e.profile.query.ProfileTaskQuery; import org.apache.skywalking.e2e.profile.query.ProfileTasks; +import org.apache.skywalking.e2e.profile.query.ProfiledSegment; +import org.apache.skywalking.e2e.profile.query.ProfiledSegmentMatcher; import org.apache.skywalking.e2e.profile.query.ProfilesTasksMatcher; import org.apache.skywalking.e2e.service.Service; import org.apache.skywalking.e2e.service.ServicesMatcher; @@ -192,6 +194,15 @@ public class ProfileVerificationITCase { } String segmentId = foundedTrace.getKey(); + + // verify segment + ProfiledSegment.ProfiledSegmentData segmentData = profileClient.getProfiledSegment(foundedTrace.getKey()); + LOGGER.info("get profiled segment : {}", segmentData); + InputStream inputStream = new ClassPathResource( + "expected-data/org.apache.skywalking.e2e.ProfileVerificationITCase.profileSegment.yml").getInputStream(); + final ProfiledSegmentMatcher tracesMatcher = new Yaml().loadAs(inputStream, ProfiledSegmentMatcher.class); + tracesMatcher.verify(segmentData); + long start = Long.parseLong(foundedTrace.getStart()); long end = start + foundedTrace.getDuration(); ProfileAnalyzation analyzation = profileClient.getProfileAnalyzation(segmentId, start, end); -- GitLab