From eb4b1215304dbab14a56b0916058b713192d5361 Mon Sep 17 00:00:00 2001 From: elk-g <37830549+elk-g@users.noreply.github.com> Date: Tue, 19 Jan 2021 08:17:29 +0800 Subject: [PATCH] Collect Log with java agent (#6127) (#6194) --- CHANGES.md | 2 +- .../log4j/v1/x/log/GRPCLogClientAppender.java | 39 +++++ .../log4j/v2/x/log/GRPCLogClientAppender.java | 53 +++++++ .../v1/x/log/GRPCLogClientAppender.java | 29 ++++ .../core/remote/LogReportServiceClient.java | 74 +++++++++ ...skywalking.apm.agent.core.boot.BootService | 3 +- .../agent/core/boot/ServiceManagerTest.java | 2 +- .../apm-toolkit-log4j-1.x-activation/pom.xml | 5 + .../v1/x/log/GRPCLogAppenderActivation.java | 75 ++++++++++ .../v1/x/log/GRPCLogAppenderInterceptor.java | 103 +++++++++++++ .../src/main/resources/skywalking-plugin.def | 1 + .../apm-toolkit-log4j-2.x-activation/pom.xml | 17 +++ .../v2/x/log/GRPCLogAppenderActivation.java | 75 ++++++++++ .../v2/x/log/GRPCLogAppenderInterceptor.java | 99 +++++++++++++ .../src/main/resources/skywalking-plugin.def | 3 +- .../pom.xml | 5 + .../v1/x/log/GRPCLogAppenderActivation.java | 75 ++++++++++ .../v1/x/log/GRPCLogAppenderInterceptor.java | 99 +++++++++++++ .../src/main/resources/skywalking-plugin.def | 3 +- .../apm-toolkit-logging-common/pom.xml | 88 +++++++++++ .../log/GRPCLogReportServiceClient.java | 140 ++++++++++++++++++ .../logging/common/log/ToolkitConfig.java | 55 +++++++ ...skywalking.apm.agent.core.boot.BootService | 19 +++ apm-sniffer/apm-toolkit-activation/pom.xml | 1 + .../Application-toolkit-log4j-1.x.md | 22 +++ .../Application-toolkit-log4j-2.x.md | 21 +++ .../Application-toolkit-logback-1.x.md | 21 +++ .../setup/service-agent/java-agent/README.md | 6 +- 28 files changed, 1129 insertions(+), 6 deletions(-) create mode 100644 apm-application-toolkit/apm-toolkit-log4j-1.x/src/main/java/org/apache/skywalking/apm/toolkit/log/log4j/v1/x/log/GRPCLogClientAppender.java create mode 100644 apm-application-toolkit/apm-toolkit-log4j-2.x/src/main/java/org/apache/skywalking/apm/toolkit/log/log4j/v2/x/log/GRPCLogClientAppender.java create mode 100644 apm-application-toolkit/apm-toolkit-logback-1.x/src/main/java/org/apache/skywalking/apm/toolkit/log/logback/v1/x/log/GRPCLogClientAppender.java create mode 100644 apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/LogReportServiceClient.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v1/x/log/GRPCLogAppenderActivation.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v1/x/log/GRPCLogAppenderInterceptor.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v2/x/log/GRPCLogAppenderActivation.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v2/x/log/GRPCLogAppenderInterceptor.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/logback/v1/x/log/GRPCLogAppenderActivation.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/logback/v1/x/log/GRPCLogAppenderInterceptor.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/pom.xml create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java create mode 100644 apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService diff --git a/CHANGES.md b/CHANGES.md index 8aa8860fed..dbc8b15590 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -35,7 +35,7 @@ Release Notes. * Fix thread leaks caused by the elasticsearch-6.x-plugin plugin. * Support reading segmentId and spanId with toolkit. * Fix RestTemplate plugin recording url tag with wrong port - +* Support collecting logs and forwarding through gRPC. #### OAP-Backend * Make meter receiver support MAL. diff --git a/apm-application-toolkit/apm-toolkit-log4j-1.x/src/main/java/org/apache/skywalking/apm/toolkit/log/log4j/v1/x/log/GRPCLogClientAppender.java b/apm-application-toolkit/apm-toolkit-log4j-1.x/src/main/java/org/apache/skywalking/apm/toolkit/log/log4j/v1/x/log/GRPCLogClientAppender.java new file mode 100644 index 0000000000..4c95b4d7d7 --- /dev/null +++ b/apm-application-toolkit/apm-toolkit-log4j-1.x/src/main/java/org/apache/skywalking/apm/toolkit/log/log4j/v1/x/log/GRPCLogClientAppender.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.log.log4j.v1.x.log; + +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.spi.LoggingEvent; + +public class GRPCLogClientAppender extends AppenderSkeleton { + @Override + protected void append(LoggingEvent loggingEvent) { + + } + + @Override + public void close() { + + } + + @Override + public boolean requiresLayout() { + return false; + } +} diff --git a/apm-application-toolkit/apm-toolkit-log4j-2.x/src/main/java/org/apache/skywalking/apm/toolkit/log/log4j/v2/x/log/GRPCLogClientAppender.java b/apm-application-toolkit/apm-toolkit-log4j-2.x/src/main/java/org/apache/skywalking/apm/toolkit/log/log4j/v2/x/log/GRPCLogClientAppender.java new file mode 100644 index 0000000000..9a09576419 --- /dev/null +++ b/apm-application-toolkit/apm-toolkit-log4j-2.x/src/main/java/org/apache/skywalking/apm/toolkit/log/log4j/v2/x/log/GRPCLogClientAppender.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.log.log4j.v2.x.log; + +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginConfiguration; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; + +@Plugin(name = "GRPCLogClientAppender", category = "Core", elementType = "appender") +public class GRPCLogClientAppender extends AbstractAppender { + + private GRPCLogClientAppender(final String name, final Filter filter, final boolean ignoreExceptions) { + super(name, filter, null, ignoreExceptions); + } + + @Override + public void append(LogEvent logEvent) { + + } + + @PluginFactory + public static GRPCLogClientAppender createAppender(@PluginAttribute("name") final String name, + @PluginElement("Filter") final Filter filter, + @PluginConfiguration final Configuration config, + @PluginAttribute("ignoreExceptions") final String ignore) { + + String appenderName = name == null ? "gRPCLogClientAppender" : name; + final boolean ignoreExceptions = "true".equalsIgnoreCase(ignore) || !"false".equalsIgnoreCase(ignore); + return new GRPCLogClientAppender(appenderName, filter, ignoreExceptions); + } +} diff --git a/apm-application-toolkit/apm-toolkit-logback-1.x/src/main/java/org/apache/skywalking/apm/toolkit/log/logback/v1/x/log/GRPCLogClientAppender.java b/apm-application-toolkit/apm-toolkit-logback-1.x/src/main/java/org/apache/skywalking/apm/toolkit/log/logback/v1/x/log/GRPCLogClientAppender.java new file mode 100644 index 0000000000..b12689f209 --- /dev/null +++ b/apm-application-toolkit/apm-toolkit-logback-1.x/src/main/java/org/apache/skywalking/apm/toolkit/log/logback/v1/x/log/GRPCLogClientAppender.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.log.logback.v1.x.log; + +import ch.qos.logback.core.AppenderBase; + +public class GRPCLogClientAppender extends AppenderBase { + + @Override + protected void append(E eventObject) { + } + +} diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/LogReportServiceClient.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/LogReportServiceClient.java new file mode 100644 index 0000000000..498af9ee69 --- /dev/null +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/LogReportServiceClient.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.agent.core.remote; + +import java.util.List; + +import org.apache.skywalking.apm.agent.core.boot.BootService; +import org.apache.skywalking.apm.agent.core.boot.DefaultImplementor; +import org.apache.skywalking.apm.commons.datacarrier.consumer.IConsumer; +import org.apache.skywalking.apm.network.logging.v3.LogData; + +@DefaultImplementor +public class LogReportServiceClient implements BootService, IConsumer { + + @Override + public void prepare() throws Throwable { + + } + + @Override + public void boot() throws Throwable { + + } + + @Override + public void onComplete() throws Throwable { + + } + + @Override + public void shutdown() throws Throwable { + + } + + @Override + public void init() { + + } + + public void produce(LogData logData) { + + } + + @Override + public void consume(List data) { + + } + + @Override + public void onError(List data, Throwable t) { + + } + + @Override + public void onExit() { + + } +} diff --git a/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService b/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService index d04e7c137f..b5ce8056e6 100644 --- a/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService +++ b/apm-sniffer/apm-agent-core/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService @@ -31,4 +31,5 @@ org.apache.skywalking.apm.agent.core.profile.ProfileSnapshotSender org.apache.skywalking.apm.agent.core.profile.ProfileTaskExecutionService org.apache.skywalking.apm.agent.core.meter.MeterService org.apache.skywalking.apm.agent.core.meter.MeterSender -org.apache.skywalking.apm.agent.core.context.status.StatusCheckService \ No newline at end of file +org.apache.skywalking.apm.agent.core.context.status.StatusCheckService +org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient \ No newline at end of file diff --git a/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java b/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java index 618610868f..da5e9bf251 100644 --- a/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java +++ b/apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/boot/ServiceManagerTest.java @@ -58,7 +58,7 @@ public class ServiceManagerTest { public void testServiceDependencies() throws Exception { HashMap registryService = getFieldValue(ServiceManager.INSTANCE, "bootedServices"); - assertThat(registryService.size(), is(16)); + assertThat(registryService.size(), is(17)); assertTraceSegmentServiceClient(ServiceManager.INSTANCE.findService(TraceSegmentServiceClient.class)); assertContextManager(ServiceManager.INSTANCE.findService(ContextManager.class)); diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/pom.xml b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/pom.xml index e0f0c713bf..d980b50dc1 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/pom.xml +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/pom.xml @@ -43,5 +43,10 @@ apm-toolkit-log4j-1.x ${project.version} + + org.apache.skywalking + apm-toolkit-logging-common + ${project.version} + diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v1/x/log/GRPCLogAppenderActivation.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v1/x/log/GRPCLogAppenderActivation.java new file mode 100644 index 0000000000..5df9eab4f9 --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v1/x/log/GRPCLogAppenderActivation.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.activation.log.log4j.v1.x.log; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static org.apache.skywalking.apm.agent.core.plugin.match.NameMatch.byName; + +import org.apache.skywalking.apm.agent.core.plugin.interceptor.ConstructorInterceptPoint; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.InstanceMethodsInterceptPoint; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.ClassInstanceMethodsEnhancePluginDefine; +import org.apache.skywalking.apm.agent.core.plugin.match.ClassMatch; + +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * enhance the method append of the grpc log send log4j class "org.apache.skywalking.apm.toolkit.log.log4j.v1.x.log + * .GRPCLogClientAppender". + */ +public class GRPCLogAppenderActivation extends ClassInstanceMethodsEnhancePluginDefine { + + public static final String INTERCEPT_CLASS = + "org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.log.GRPCLogAppenderInterceptor"; + public static final String ENHANCE_CLASS = + "org.apache.skywalking.apm.toolkit.log.log4j.v1.x.log.GRPCLogClientAppender"; + public static final String ENHANCE_METHOD = "append"; + + @Override + protected ClassMatch enhanceClass() { + return byName(ENHANCE_CLASS); + } + + @Override + public ConstructorInterceptPoint[] getConstructorsInterceptPoints() { + return null; + } + + @Override + public InstanceMethodsInterceptPoint[] getInstanceMethodsInterceptPoints() { + return new InstanceMethodsInterceptPoint[] { + new InstanceMethodsInterceptPoint() { + @Override + public ElementMatcher getMethodsMatcher() { + return named(ENHANCE_METHOD); + } + + @Override + public String getMethodsInterceptor() { + return INTERCEPT_CLASS; + } + + @Override + public boolean isOverrideArgs() { + return false; + } + } + }; + } +} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v1/x/log/GRPCLogAppenderInterceptor.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v1/x/log/GRPCLogAppenderInterceptor.java new file mode 100644 index 0000000000..b93fd152a0 --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v1/x/log/GRPCLogAppenderInterceptor.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.activation.log.log4j.v1.x.log; + +import java.lang.reflect.Method; +import java.util.Objects; + +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.context.ContextManager; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.EnhancedInstance; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstanceMethodsAroundInterceptor; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.MethodInterceptResult; +import org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogDataBody; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.apm.network.logging.v3.TextLog; +import org.apache.skywalking.apm.network.logging.v3.TraceContext; +import org.slf4j.event.LoggingEvent; + +public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundInterceptor { + + private LogReportServiceClient client; + + @Override + public void beforeMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class[] argumentsTypes, + MethodInterceptResult result) throws Throwable { + if (Objects.isNull(client)) { + client = ServiceManager.INSTANCE.findService(LogReportServiceClient.class); + if (Objects.isNull(client)) { + return; + } + } + LoggingEvent event = (LoggingEvent) allArguments[0]; + if (Objects.nonNull(event)) { + client.produce(transform(event)); + } + } + + @Override + public Object afterMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class[] argumentsTypes, + Object ret) throws Throwable { + return ret; + } + + @Override + public void handleMethodException(EnhancedInstance objInst, Method method, Object[] allArguments, + Class[] argumentsTypes, Throwable t) { + + } + + /** + * transforms {@link LoggingEvent} to {@link LogData} + * + * @param event {@link LoggingEvent} + * @return {@link LogData} with filtered trace context in order to reduce the cost on the network + */ + private LogData transform(LoggingEvent event) { + LogData.Builder builder = LogData.newBuilder() + .setTimestamp(event.getTimeStamp()) + .setService(Config.Agent.SERVICE_NAME) + .setServiceInstance(Config.Agent.INSTANCE_NAME) + .setTraceContext(TraceContext.newBuilder() + .setTraceId(ContextManager.getGlobalTraceId()) + .setSpanId(ContextManager.getSpanId()) + .setTraceSegmentId(ContextManager.getSegmentId()) + .build()) + .setTags(LogTags.newBuilder() + .addData(KeyStringValuePair.newBuilder() + .setKey("level").setValue(event.getLevel().toString()).build()) + .addData(KeyStringValuePair.newBuilder() + .setKey("logger").setValue(event.getLoggerName()).build()) + .addData(KeyStringValuePair.newBuilder() + .setKey("thread").setValue(event.getThreadName()).build()) + .build()) + .setBody(LogDataBody.newBuilder().setType(LogDataBody.ContentCase.TEXT.name()) + .setText(TextLog.newBuilder().setText(event.getMessage()).build()).build()); + return -1 == ContextManager.getSpanId() ? builder.build() + : builder.setTraceContext(TraceContext.newBuilder() + .setTraceId(ContextManager.getGlobalTraceId()) + .setSpanId(ContextManager.getSpanId()) + .setTraceSegmentId(ContextManager.getSegmentId()) + .build()).build(); + } +} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/resources/skywalking-plugin.def b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/resources/skywalking-plugin.def index e66f8e127a..1de7a35586 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/resources/skywalking-plugin.def +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-1.x-activation/src/main/resources/skywalking-plugin.def @@ -15,3 +15,4 @@ # limitations under the License. toolkit-log4j=org.apache.skywalking.apm.toolkit.activation.log.log4j.v1.x.TraceIdPatternConverterActivation +toolkit-log4j=org.apache.skywalking.apm.toolkit.activation.log.log4j.v1.x.log.GRPCLogAppenderActivation diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/pom.xml b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/pom.xml index 8255bd6435..fceb08f009 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/pom.xml +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/pom.xml @@ -27,5 +27,22 @@ apm-toolkit-log4j-2.x-activation + + 2.7 + + + + + org.apache.logging.log4j + log4j-core + ${log4j-core.version} + provided + + + org.apache.skywalking + apm-toolkit-logging-common + ${project.version} + + diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v2/x/log/GRPCLogAppenderActivation.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v2/x/log/GRPCLogAppenderActivation.java new file mode 100644 index 0000000000..21ec38300d --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v2/x/log/GRPCLogAppenderActivation.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.log; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static org.apache.skywalking.apm.agent.core.plugin.match.NameMatch.byName; + +import org.apache.skywalking.apm.agent.core.plugin.interceptor.ConstructorInterceptPoint; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.InstanceMethodsInterceptPoint; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.ClassInstanceMethodsEnhancePluginDefine; +import org.apache.skywalking.apm.agent.core.plugin.match.ClassMatch; + +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * enhance the method append of the grpc log send lo4j2 class "org.apache.skywalking.apm.toolkit.log.log4j.v2.x.log + * .GRPCLogClientAppender". + */ +public class GRPCLogAppenderActivation extends ClassInstanceMethodsEnhancePluginDefine { + + public static final String INTERCEPT_CLASS = + "org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.log.GRPCLogAppenderInterceptor"; + public static final String ENHANCE_CLASS = + "org.apache.skywalking.apm.toolkit.log.log4j.v2.x.log.GRPCLogClientAppender"; + public static final String ENHANCE_METHOD = "append"; + + @Override + protected ClassMatch enhanceClass() { + return byName(ENHANCE_CLASS); + } + + @Override + public ConstructorInterceptPoint[] getConstructorsInterceptPoints() { + return null; + } + + @Override + public InstanceMethodsInterceptPoint[] getInstanceMethodsInterceptPoints() { + return new InstanceMethodsInterceptPoint[] { + new InstanceMethodsInterceptPoint() { + @Override + public ElementMatcher getMethodsMatcher() { + return named(ENHANCE_METHOD); + } + + @Override + public String getMethodsInterceptor() { + return INTERCEPT_CLASS; + } + + @Override + public boolean isOverrideArgs() { + return false; + } + } + }; + } +} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v2/x/log/GRPCLogAppenderInterceptor.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v2/x/log/GRPCLogAppenderInterceptor.java new file mode 100644 index 0000000000..9ab34466a3 --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/log4j/v2/x/log/GRPCLogAppenderInterceptor.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.log; + +import java.lang.reflect.Method; +import java.util.Objects; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.context.ContextManager; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.EnhancedInstance; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstanceMethodsAroundInterceptor; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.MethodInterceptResult; +import org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogDataBody; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.apm.network.logging.v3.TextLog; +import org.apache.skywalking.apm.network.logging.v3.TraceContext; + +public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundInterceptor { + + private LogReportServiceClient client; + + @Override + public void beforeMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class[] argumentsTypes, + MethodInterceptResult result) throws Throwable { + if (Objects.isNull(client)) { + client = ServiceManager.INSTANCE.findService(LogReportServiceClient.class); + if (Objects.isNull(client)) { + return; + } + } + LogEvent event = (LogEvent) allArguments[0]; + if (Objects.nonNull(event)) { + client.produce(transform(event)); + } + } + + @Override + public Object afterMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class[] argumentsTypes, + Object ret) throws Throwable { + return ret; + } + + @Override + public void handleMethodException(EnhancedInstance objInst, Method method, Object[] allArguments, + Class[] argumentsTypes, Throwable t) { + + } + + /** + * transforms {@link LogEvent} to {@link LogData} + * + * @param event {@link LogEvent} + * @return {@link LogData} with filtered trace context in order to reduce the cost on the network + */ + private LogData transform(LogEvent event) { + LogData.Builder builder = LogData.newBuilder() + .setTimestamp(event.getTimeMillis()) + .setService(Config.Agent.SERVICE_NAME) + .setServiceInstance(Config.Agent.INSTANCE_NAME) + .setTags(LogTags.newBuilder() + .addData(KeyStringValuePair.newBuilder() + .setKey("level").setValue(event.getLevel().toString()).build()) + .addData(KeyStringValuePair.newBuilder() + .setKey("logger").setValue(event.getLoggerName()).build()) + .addData(KeyStringValuePair.newBuilder() + .setKey("thread").setValue(event.getThreadName()).build()) + .build()) + .setBody(LogDataBody.newBuilder().setType(LogDataBody.ContentCase.TEXT.name()) + .setText(TextLog.newBuilder().setText(event.getMessage().getFormattedMessage()).build()) + .build()); + return -1 == ContextManager.getSpanId() ? builder.build() + : builder.setTraceContext(TraceContext.newBuilder() + .setTraceId(ContextManager.getGlobalTraceId()) + .setSpanId(ContextManager.getSpanId()) + .setTraceSegmentId(ContextManager.getSegmentId()) + .build()).build(); + } +} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/resources/skywalking-plugin.def b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/resources/skywalking-plugin.def index 159d36e37a..f8d8c72148 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/resources/skywalking-plugin.def +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-log4j-2.x-activation/src/main/resources/skywalking-plugin.def @@ -18,4 +18,5 @@ toolkit-log4j2=org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.Trace toolkit-log4j2=org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.async.AsyncLoggerConfigInstrumentation toolkit-log4j2=org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.async.Log4jLogEventInstrumentation toolkit-log4j2=org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.async.RingBufferLogEventInstrumentation -toolkit-log4j2=org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.async.AsyncAppenderInstrumentation \ No newline at end of file +toolkit-log4j2=org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.async.AsyncAppenderInstrumentation +toolkit-log4j2=org.apache.skywalking.apm.toolkit.activation.log.log4j.v2.x.log.GRPCLogAppenderActivation \ No newline at end of file diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/pom.xml b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/pom.xml index 969a2689ca..b1fd097778 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/pom.xml +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/pom.xml @@ -38,5 +38,10 @@ ${logback-classic.version} provided + + org.apache.skywalking + apm-toolkit-logging-common + ${project.version} + diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/logback/v1/x/log/GRPCLogAppenderActivation.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/logback/v1/x/log/GRPCLogAppenderActivation.java new file mode 100644 index 0000000000..8b4fbec1cb --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/logback/v1/x/log/GRPCLogAppenderActivation.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.log; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static org.apache.skywalking.apm.agent.core.plugin.match.NameMatch.byName; + +import org.apache.skywalking.apm.agent.core.plugin.interceptor.ConstructorInterceptPoint; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.InstanceMethodsInterceptPoint; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.ClassInstanceMethodsEnhancePluginDefine; +import org.apache.skywalking.apm.agent.core.plugin.match.ClassMatch; + +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * enhance the method append of the grpc log send logback class "org.apache.skywalking.apm.toolkit + * .log.logback.v1.x.log.GRPCLogClientAppender". + */ +public class GRPCLogAppenderActivation extends ClassInstanceMethodsEnhancePluginDefine { + + public static final String INTERCEPT_CLASS = + "org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.log.GRPCLogAppenderInterceptor"; + public static final String ENHANCE_CLASS = + "org.apache.skywalking.apm.toolkit.log.logback.v1.x.log.GRPCLogClientAppender"; + public static final String ENHANCE_METHOD = "append"; + + @Override + protected ClassMatch enhanceClass() { + return byName(ENHANCE_CLASS); + } + + @Override + public ConstructorInterceptPoint[] getConstructorsInterceptPoints() { + return null; + } + + @Override + public InstanceMethodsInterceptPoint[] getInstanceMethodsInterceptPoints() { + return new InstanceMethodsInterceptPoint[] { + new InstanceMethodsInterceptPoint() { + @Override + public ElementMatcher getMethodsMatcher() { + return named(ENHANCE_METHOD); + } + + @Override + public String getMethodsInterceptor() { + return INTERCEPT_CLASS; + } + + @Override + public boolean isOverrideArgs() { + return false; + } + } + }; + } +} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/logback/v1/x/log/GRPCLogAppenderInterceptor.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/logback/v1/x/log/GRPCLogAppenderInterceptor.java new file mode 100644 index 0000000000..529c259c77 --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/java/org/apache/skywalking/apm/toolkit/activation/log/logback/v1/x/log/GRPCLogAppenderInterceptor.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.log; + +import java.lang.reflect.Method; +import java.util.Objects; + +import org.apache.skywalking.apm.agent.core.boot.ServiceManager; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.context.ContextManager; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.EnhancedInstance; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstanceMethodsAroundInterceptor; +import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.MethodInterceptResult; +import org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogDataBody; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.apm.network.logging.v3.TextLog; +import org.apache.skywalking.apm.network.logging.v3.TraceContext; + +import ch.qos.logback.classic.spi.ILoggingEvent; + +public class GRPCLogAppenderInterceptor implements InstanceMethodsAroundInterceptor { + + private LogReportServiceClient client; + + @Override + public void beforeMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class[] argumentsTypes, + MethodInterceptResult result) throws Throwable { + if (Objects.isNull(client)) { + client = ServiceManager.INSTANCE.findService(LogReportServiceClient.class); + if (Objects.isNull(client)) { + return; + } + } + ILoggingEvent event = (ILoggingEvent) allArguments[0]; + if (Objects.nonNull(event)) { + client.produce(transform(event)); + } + } + + @Override + public Object afterMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class[] argumentsTypes, + Object ret) throws Throwable { + return ret; + } + + @Override + public void handleMethodException(EnhancedInstance objInst, Method method, Object[] allArguments, + Class[] argumentsTypes, Throwable t) { + + } + + /** + * transforms {@link ILoggingEvent} to {@link LogData} + * + * @param event {@link ILoggingEvent} + * @return {@link LogData} with filtered trace context in order to reduce the cost on the network + */ + private LogData transform(ILoggingEvent event) { + LogData.Builder builder = LogData.newBuilder() + .setTimestamp(event.getTimeStamp()) + .setService(Config.Agent.SERVICE_NAME) + .setServiceInstance(Config.Agent.INSTANCE_NAME) + .setTags(LogTags.newBuilder() + .addData(KeyStringValuePair.newBuilder() + .setKey("level").setValue(event.getLevel().toString()).build()) + .addData(KeyStringValuePair.newBuilder() + .setKey("logger").setValue(event.getLoggerName()).build()) + .addData(KeyStringValuePair.newBuilder() + .setKey("thread").setValue(event.getThreadName()).build()) + .build()) + .setBody(LogDataBody.newBuilder().setType(LogDataBody.ContentCase.TEXT.name()) + .setText(TextLog.newBuilder().setText(event.getFormattedMessage()).build()).build()); + return -1 == ContextManager.getSpanId() ? builder.build() + : builder.setTraceContext(TraceContext.newBuilder() + .setTraceId(ContextManager.getGlobalTraceId()) + .setSpanId(ContextManager.getSpanId()) + .setTraceSegmentId(ContextManager.getSegmentId()) + .build()).build(); + } +} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/resources/skywalking-plugin.def b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/resources/skywalking-plugin.def index 41140633c3..385c919132 100644 --- a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/resources/skywalking-plugin.def +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logback-1.x-activation/src/main/resources/skywalking-plugin.def @@ -19,4 +19,5 @@ toolkit-logback=org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.md toolkit-logback=org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.async.AsyncAppenderBaseInstrumentation toolkit-logback=org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.async.LoggingEventInstrumentation toolkit-logback=org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.logstash.TcpSocketAppenderActivation -toolkit-logback=org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.logstash.TraceIdJsonProviderActivation \ No newline at end of file +toolkit-logback=org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.logstash.TraceIdJsonProviderActivation +toolkit-logback=org.apache.skywalking.apm.toolkit.activation.log.logback.v1.x.log.GRPCLogAppenderActivation \ No newline at end of file diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/pom.xml b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/pom.xml new file mode 100644 index 0000000000..368245c958 --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/pom.xml @@ -0,0 +1,88 @@ + + + + + apm-toolkit-activation + org.apache.skywalking + 8.4.0-SNAPSHOT + + 4.0.0 + + apm-toolkit-logging-common + + + org.apache.skywalking.apm.dependencies + com.google + ${toolkit.shade.package}.${shade.com.google.source} + io.grpc + ${toolkit.shade.package}.${shade.io.grpc.source} + io.netty + ${toolkit.shade.package}.${shade.io.netty.source} + + + + + org.apache.skywalking + apm-network + ${project.version} + + + + + + + maven-shade-plugin + + + package + + shade + + + + + com.google.errorprone:error_prone_annotations:jar: + com.google.code.findbugs:jsr305:jar: + com.google.android:annotations:jar: + com.google.api.grpc:proto-google-common-protos:jar: + + + + + ${shade.com.google.source} + ${shade.com.google.target} + + + ${shade.io.grpc.source} + ${shade.io.grpc.target} + + + ${shade.io.netty.source} + ${shade.io.netty.target} + + + + + + + + + + diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java new file mode 100644 index 0000000000..568c50817a --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/GRPCLogReportServiceClient.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.logging.common.log; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.skywalking.apm.agent.core.boot.OverrideImplementor; +import org.apache.skywalking.apm.agent.core.conf.Config; +import org.apache.skywalking.apm.agent.core.logging.api.ILog; +import org.apache.skywalking.apm.agent.core.logging.api.LogManager; +import org.apache.skywalking.apm.agent.core.remote.GRPCStreamServiceStatus; +import org.apache.skywalking.apm.agent.core.remote.LogReportServiceClient; +import org.apache.skywalking.apm.agent.core.util.CollectionUtil; +import org.apache.skywalking.apm.commons.datacarrier.DataCarrier; +import org.apache.skywalking.apm.commons.datacarrier.buffer.BufferStrategy; +import org.apache.skywalking.apm.network.common.v3.Commands; +import org.apache.skywalking.apm.network.logging.v3.LogData; +import org.apache.skywalking.apm.network.logging.v3.LogReportServiceGrpc; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.StreamObserver; + +/** + * Report log to server by grpc + */ +@OverrideImplementor(LogReportServiceClient.class) +public class GRPCLogReportServiceClient extends LogReportServiceClient { + + private static final ILog LOGGER = LogManager.getLogger(GRPCLogReportServiceClient.class); + + private volatile DataCarrier carrier; + + private LogReportServiceGrpc.LogReportServiceStub asyncStub; + + private ManagedChannel channel; + + private AtomicBoolean disconnected = new AtomicBoolean(false); + + @Override + public void boot() throws Throwable { + carrier = new DataCarrier<>("gRPC-log", "gRPC-log", Config.Buffer.CHANNEL_SIZE, Config.Buffer.BUFFER_SIZE, + BufferStrategy.IF_POSSIBLE); + carrier.consume(this, 1); + channel = ManagedChannelBuilder + .forAddress(ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_HOST, + ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.SERVER_PORT).usePlaintext().build(); + asyncStub = LogReportServiceGrpc.newStub(channel) + .withMaxOutboundMessageSize(ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.MAX_MESSAGE_SIZE); + } + + @Override + public void shutdown() { + try { + carrier.shutdownConsumers(); + if (channel != null) { + channel.shutdownNow(); + } + } catch (Throwable t) { + LOGGER.error(t.getMessage(), t); + } + } + + @Override + public void produce(LogData logData) { + if (Objects.nonNull(logData) && !carrier.produce(logData)) { + if (LOGGER.isDebugEnable()) { + LOGGER.debug("One log has been abandoned, cause by buffer is full."); + } + } + } + + @Override + public void consume(final List dataList) { + if (CollectionUtil.isEmpty(dataList)) { + return; + } + StreamObserver reportStreamObserver = null; + final GRPCStreamServiceStatus waitStatus = new GRPCStreamServiceStatus(false); + try { + reportStreamObserver = asyncStub.withDeadlineAfter( + ToolkitConfig.Plugin.Toolkit.Log.GRPC.Reporter.UPSTREAM_TIMEOUT, TimeUnit.SECONDS + ).collect(new StreamObserver() { + @Override + public void onNext(Commands commands) { + } + + @Override + public void onError(Throwable t) { + waitStatus.finished(); + if (disconnected.compareAndSet(false, true)) { + LOGGER.error("Send log to gRPC server fail with an internal exception.", t); + } + + LOGGER.error(t, "Try to send {} log data to collector, with unexpected exception.", + dataList.size()); + } + + @Override + public void onCompleted() { + disconnected.compareAndSet(true, false); + waitStatus.finished(); + } + }); + + for (final LogData logData : dataList) { + reportStreamObserver.onNext(logData); + } + } catch (Throwable e) { + if (!(e instanceof StatusRuntimeException)) { + LOGGER.error(e, "Report log failure with the gRPC client."); + } + } finally { + if (reportStreamObserver != null) { + reportStreamObserver.onCompleted(); + } + waitStatus.wait4Finish(); + } + } +} diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java new file mode 100644 index 0000000000..c0bc233b90 --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/java/org/apache/skywalking/apm/toolkit/logging/common/log/ToolkitConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.apm.toolkit.logging.common.log; + +import org.apache.skywalking.apm.agent.core.boot.PluginConfig; + +public class ToolkitConfig { + + public static class Plugin { + @PluginConfig(root = ToolkitConfig.class) + public static class Toolkit { + public static class Log { + public static class GRPC { + public static class Reporter { + /** + * The host of gRPC log server. + */ + public static String SERVER_HOST = "127.0.0.1"; + + /** + * The port of gRPC log server. + */ + public static int SERVER_PORT = 11800; + + /** + * The max size of message to send to server.Default is 10 MB. + */ + public static int MAX_MESSAGE_SIZE = 10 * 1024 * 1024; + + /** + * How long grpc client will timeout in sending data to upstream. The unit is second. + */ + public static int UPSTREAM_TIMEOUT = 30; + } + } + } + } + } +} \ No newline at end of file diff --git a/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService new file mode 100644 index 0000000000..69de3fb6b0 --- /dev/null +++ b/apm-sniffer/apm-toolkit-activation/apm-toolkit-logging-common/src/main/resources/META-INF/services/org.apache.skywalking.apm.agent.core.boot.BootService @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.apm.toolkit.logging.common.log.GRPCLogReportServiceClient \ No newline at end of file diff --git a/apm-sniffer/apm-toolkit-activation/pom.xml b/apm-sniffer/apm-toolkit-activation/pom.xml index 27ac7db495..3a3bccdbd2 100644 --- a/apm-sniffer/apm-toolkit-activation/pom.xml +++ b/apm-sniffer/apm-toolkit-activation/pom.xml @@ -33,6 +33,7 @@ apm-toolkit-opentracing-activation apm-toolkit-trace-activation apm-toolkit-meter-activation + apm-toolkit-logging-common apm-toolkit-activation diff --git a/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-1.x.md b/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-1.x.md index d6ab5a5c1a..1a0f3ef228 100644 --- a/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-1.x.md +++ b/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-1.x.md @@ -7,6 +7,8 @@ ``` +# Print trace ID in your logs + * Config a layout ```properties log4j.appender.CONSOLE.layout=TraceIdPatternLayout @@ -18,3 +20,23 @@ log4j.appender.CONSOLE.layout.ConversionPattern=%d [%T] %-5p %c{1}:%L - %m%n ``` * When you use `-javaagent` to active the sky-walking tracer, log4j will output **traceId**, if it existed. If the tracer is inactive, the output will be `TID: N/A`. + +# gRPC reporter + +The gRPC report could forward the collected logs to SkyWalking OAP server, or [SkyWalking Satellite sidecar](https://github.com/apache/skywalking-satellite). Trace id, segment id, and span id will attach to logs automatically. You don't need to change the layout. + +* Add `GRPCLogClientAppender` in log4j.properties + +```properties +log4j.rootLogger=INFO,CustomAppender +log4j.appender.CustomAppender=org.apache.skywalking.apm.toolkit.log.log4j.v1.x.log.GRPCLogClientAppender +``` + +* Add config of the plugin or use default + +```properties +plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:127.0.0.1} +plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800} +plugin.toolkit.log.grpc.reporter.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} +plugin.toolkit.log.grpc.reporter.upstream_timeout=${SW_GRPC_LOG_GRPC_UPSTREAM_TIMEOUT:30} +``` diff --git a/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-2.x.md b/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-2.x.md index 3a87efa8e6..00f4ed1d38 100644 --- a/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-2.x.md +++ b/docs/en/setup/service-agent/java-agent/Application-toolkit-log4j-2.x.md @@ -7,6 +7,8 @@ ``` +# Print trace ID in your logs + * Config the `[%traceId]` pattern in your log4j2.xml ```xml @@ -101,3 +103,22 @@ ``` * When you use `-javaagent` to active the sky-walking tracer, log4j2 will output **traceId**, if it existed. If the tracer is inactive, the output will be `TID: N/A`. + +# gRPC reporter + +The gRPC report could forward the collected logs to SkyWalking OAP server, or [SkyWalking Satellite sidecar](https://github.com/apache/skywalking-satellite). Trace id, segment id, and span id will attach to logs automatically. You don't need to change the layout. + +* Add `GRPCLogClientAppender` in log4j2.xml + +```xml + +``` + +* Add config of the plugin or use default + +```properties +plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:127.0.0.1} +plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800} +plugin.toolkit.log.grpc.reporter.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} +plugin.toolkit.log.grpc.reporter.upstream_timeout=${SW_GRPC_LOG_GRPC_UPSTREAM_TIMEOUT:30} +``` \ No newline at end of file diff --git a/docs/en/setup/service-agent/java-agent/Application-toolkit-logback-1.x.md b/docs/en/setup/service-agent/java-agent/Application-toolkit-logback-1.x.md index 1e1493bc7a..0c44478b26 100644 --- a/docs/en/setup/service-agent/java-agent/Application-toolkit-logback-1.x.md +++ b/docs/en/setup/service-agent/java-agent/Application-toolkit-logback-1.x.md @@ -8,6 +8,8 @@ ``` +# Print trace ID in your logs + * set `%tid` in `Pattern` section of logback.xml ```xml @@ -108,3 +110,22 @@ ``` + +# gRPC reporter + +The gRPC report could forward the collected logs to SkyWalking OAP server, or [SkyWalking Satellite sidecar](https://github.com/apache/skywalking-satellite). Trace id, segment id, and span id will attach to logs automatically. You don't need to change the layout. + +* Add `GRPCLogClientAppender` in logback.xml + +```xml + +``` + +* Add config of the plugin or use default + +```properties +plugin.toolkit.log.grpc.reporter.server_host=${SW_GRPC_LOG_SERVER_HOST:127.0.0.1} +plugin.toolkit.log.grpc.reporter.server_port=${SW_GRPC_LOG_SERVER_PORT:11800} +plugin.toolkit.log.grpc.reporter.max_message_size=${SW_GRPC_LOG_MAX_MESSAGE_SIZE:10485760} +plugin.toolkit.log.grpc.reporter.upstream_timeout=${SW_GRPC_LOG_GRPC_UPSTREAM_TIMEOUT:30} +``` \ No newline at end of file diff --git a/docs/en/setup/service-agent/java-agent/README.md b/docs/en/setup/service-agent/java-agent/README.md index a383302a65..f2811d6dd6 100755 --- a/docs/en/setup/service-agent/java-agent/README.md +++ b/docs/en/setup/service-agent/java-agent/README.md @@ -159,6 +159,10 @@ property key | Description | Default | `plugin.kafka.topic_profilings` | Specify which Kafka topic name for Thread Profiling snapshot to report to. | `skywalking_profilings` | `plugin.kafka.topic_management` | Specify which Kafka topic name for the register or heartbeat data of Service Instance to report to. | `skywalking_managements` | `plugin.springannotation.classname_match_regex` | Match spring beans with regular expression for the class name. Multiple expressions could be separated by a comma. This only works when `Spring annotation plugin` has been activated. | `All the spring beans tagged with @Bean,@Service,@Dao, or @Repository.` | +`plugin.toolkit.log.grpc.reporter.server_host` | Specify which grpc server's host for log data to report to. | `127.0.0.1` | +`plugin.toolkit.log.grpc.reporter.server_port` | Specify which grpc server's port for log data to report to. | `11800` | +`plugin.toolkit.log.grpc.reporter.max_message_size` | Specify the maximum size of log data for grpc client to report to. | `10485760` | +`plugin.toolkit.log.grpc.reporter.upstream_timeout` | How long grpc client will timeout in sending data to upstream. Unit is second.|`30` seconds| ## Optional Plugins Java agent plugins are all pluggable. Optional plugins could be provided in `optional-plugins` folder under agent or 3rd party repositories. @@ -207,7 +211,7 @@ The tag is, key=`x-le` and value = `{"logic-span":true}`. * Application Toolkit, are a collection of libraries, provided by SkyWalking APM. Using them, you have a bridge between your application and SkyWalking APM agent. * If you want your codes to interact with SkyWalking agent, including `getting trace id`, `setting tags`, `propagating custom data` etc.. Try [SkyWalking manual APIs](Application-toolkit-trace.md). * If you require customized metrics, try [SkyWalking Meter System Toolkit](Application-toolkit-meter.md). - * If you want to print trace context(e.g. traceId) in your logs, choose the log frameworks, [log4j](Application-toolkit-log4j-1.x.md), + * If you want to print trace context(e.g. traceId) in your logs, or collect logs, choose the log frameworks, [log4j](Application-toolkit-log4j-1.x.md), [log4j2](Application-toolkit-log4j-2.x.md), [logback](Application-toolkit-logback-1.x.md) * If you want to continue traces across thread manually, use [across thread solution APIs](Application-toolkit-trace-cross-thread.md). * If you want to forward MicroMeter/Spring Sleuth metrics to Meter System, use [SkyWalking MicroMeter Register](Application-toolkit-micrometer.md). -- GitLab