提交 72b18f89 编写于 作者: 如梦技术's avatar 如梦技术 🐛

mica-logging 完成 loki #36 #I3PX2F

上级 fd663736
......@@ -18,6 +18,7 @@ ext {
lombokVersion = "1.18.20"
findbugsVersion = "3.0.2"
logstashVersion = "6.6"
lokiLogbackAppender = "1.2.0"
zxingVersion = "3.4.1"
jetCacheVersion = "2.6.0"
}
......
......@@ -3,7 +3,7 @@
## 功能扩展
- jackson KeyConvertorParser、encoder、decoder
- 添加 spring-configuration-metadata.json
- 扩展 metrics 打通 micrometer(暂时未实现)
- 扩展 metrics 打通 micrometer
## 使用
### maven
......@@ -20,6 +20,13 @@
compile("net.dreamlu:mica-jetcache:${version}")
```
## 配置项
| 配置项 | 默认值 | 说明 |
| ----- | ------ | ------ |
| jetcache.metrics.enabled | true | 开启 jetcache metrics,默认:true |
| jetcache.metrics.enabled-stat-info-logger | false | 开启 StatInfoLogger |
| jetcache.metrics.verbose-log | false | StatInfoLogger 打印明细,默认:false |
## 配置示例
```yaml
jetcache:
......
......@@ -21,12 +21,14 @@ import net.dreamlu.mica.core.exception.ServiceException;
import net.dreamlu.mica.core.result.R;
import net.dreamlu.mica.core.result.SystemCode;
import net.dreamlu.mica.core.utils.BeanUtil;
import net.dreamlu.mica.core.utils.StringUtil;
import org.springframework.boot.web.error.ErrorAttributeOptions;
import org.springframework.boot.web.servlet.error.DefaultErrorAttributes;
import org.springframework.lang.Nullable;
import org.springframework.web.context.request.RequestAttributes;
import org.springframework.web.context.request.WebRequest;
import javax.servlet.RequestDispatcher;
import java.util.Map;
import java.util.Optional;
......@@ -41,8 +43,14 @@ public class MicaErrorAttributes extends DefaultErrorAttributes {
@Override
public Map<String, Object> getErrorAttributes(WebRequest webRequest, ErrorAttributeOptions options) {
String requestUrl = this.getAttr(webRequest, "javax.servlet.error.request_uri");
Integer status = this.getAttr(webRequest, "javax.servlet.error.status_code");
// 请求地址
String requestUrl = this.getAttr(webRequest, RequestDispatcher.ERROR_REQUEST_URI);
if (StringUtil.isBlank(requestUrl)) {
requestUrl = this.getAttr(webRequest, RequestDispatcher.FORWARD_REQUEST_URI);
}
// status code
Integer status = this.getAttr(webRequest, RequestDispatcher.ERROR_STATUS_CODE);
// error
Throwable error = getError(webRequest);
log.error("URL:{} error status:{}", requestUrl, status, error);
R<Object> result;
......
......@@ -28,7 +28,7 @@ compile("net.dreamlu:mica-logging:${version}")
```
## 可选依赖
**注意:** 开启 `json` 文件或 `logstash` **必须添加**该依赖!!!
- 开启 `json` 文件或 `logstash` **必须添加**该依赖!!!
```xml
<dependency>
......@@ -38,6 +38,15 @@ compile("net.dreamlu:mica-logging:${version}")
</dependency>
```
- 开启 `logstash` 额外还需要得依赖项
```xml
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>${version}</version>
</dependency>
```
## 配置
**特别注意:** 需要配置`服务名``环境`,例如:
......@@ -59,6 +68,34 @@ spring:
| mica.logging.logstash.port | 5000 | logstash port |
| mica.logging.logstash.queue-size | 512 | logstash 队列大小 |
### loki 配置项
| 配置项 | 默认值 | 说明 |
| ----- | ------ | ------ |
| mica.logging.loki.batch-max-bytes | 0 | |
| mica.logging.loki.batch-max-items | 1000 | 通用配置 |
| mica.logging.loki.batch-timeout-ms | 60000 | |
| mica.logging.loki.drain-on-stop | true | |
| mica.logging.loki.enabled | false | 是否开启 loki 日志收集 |
| mica.logging.loki.encoder | | 编码方式 |
| mica.logging.loki.format-label-key-value-separator | = | |
| mica.logging.loki.format-label-no-pex | true | |
| mica.logging.loki.format-label-pair-separator | , | |
| mica.logging.loki.format-label-pattern | | format 配置 |
| mica.logging.loki.format-message-pattern | | |
| mica.logging.loki.format-sort-by-time | false | |
| mica.logging.loki.format-static-labels | false | |
| mica.logging.loki.http-auth-password | | |
| mica.logging.loki.http-auth-tenant-id | | |
| mica.logging.loki.http-auth-username | | |
| mica.logging.loki.http-connection-timeout-ms | 30000 | |
| mica.logging.loki.http-request-timeout-ms | 5000 | |
| mica.logging.loki.http-sender | | http sender,默认: java11 |
| mica.logging.loki.http-url | | http 配置 |
| mica.logging.loki.metrics-enabled | false | 开启 metrics |
| mica.logging.loki.send-queue-max-bytes | 41943040 | |
| mica.logging.loki.use-direct-buffers | true | 使用堆外内存 |
| mica.logging.loki.verbose | false | |
## 日志示例
### 文件日志
```shell
......
......@@ -2,8 +2,10 @@ dependencies {
api project(":mica-core")
api "org.springframework.boot:spring-boot-starter-logging"
implementation "net.logstash.logback:logstash-logback-encoder:${logstashVersion}"
implementation "com.github.loki4j:loki-logback-appender:${lokiLogbackAppender}"
implementation "org.springframework.boot:spring-boot-autoconfigure"
compileOnly "org.springframework.cloud:spring-cloud-context"
compileOnly "com.squareup.okhttp3:okhttp"
compileOnly "net.dreamlu:mica-auto:${micaAutoVersion}"
annotationProcessor "net.dreamlu:mica-auto:${micaAutoVersion}"
}
......@@ -22,6 +22,10 @@ public enum Appender {
/**
* logstash
*/
LOG_STASH;
LOG_STASH,
/**
* loki
*/
LOKI;
}
......@@ -28,7 +28,6 @@ import net.logstash.logback.encoder.LogstashEncoder;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.Map;
......@@ -84,12 +83,12 @@ public class LoggingLogStashAppender implements ILoggingAppender {
* @param customFields a {@link String} object.
* @param logStashProperties a {@link net.dreamlu.mica.logging.config.MicaLoggingProperties.Logstash} object.
*/
public static void addLogStashTcpSocketAppender(LoggerContext context,
private static void addLogStashTcpSocketAppender(LoggerContext context,
String customFields,
MicaLoggingProperties.Logstash logStashProperties) {
// More documentation is available at: https://github.com/logstash/logstash-logback-encoder
final LogstashTcpSocketAppender logStashAppender = new LogstashTcpSocketAppender();
logStashAppender.addDestinations(new InetSocketAddress(logStashProperties.getHost(), logStashProperties.getPort()));
logStashAppender.addDestination(logStashProperties.getDestinations());
logStashAppender.setContext(context);
logStashAppender.setEncoder(logstashEncoder(customFields));
logStashAppender.setName(ASYNC_LOG_STASH_APPENDER_NAME);
......
/*
* Copyright (c) 2019-2029, Dreamlu 卢春梦 (596392912@qq.com & www.dreamlu.net).
* <p>
* Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.gnu.org/licenses/lgpl.html
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.dreamlu.mica.logging.appender;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import com.github.loki4j.logback.*;
import lombok.extern.slf4j.Slf4j;
import net.dreamlu.mica.core.utils.StringUtil;
import net.dreamlu.mica.logging.config.MicaLoggingProperties;
import net.dreamlu.mica.logging.loki.OkHttpSender;
import org.slf4j.LoggerFactory;
/**
* loki 日志接收
*
* @author L.cm
*/
@Slf4j
public class LoggingLokiAppender implements ILoggingAppender {
private static final String APPENDER_NAME = "LOKI";
private final MicaLoggingProperties properties;
public LoggingLokiAppender(MicaLoggingProperties properties) {
this.properties = properties;
LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
this.start(context);
}
@Override
public void start(LoggerContext context) {
log.info("LogStash logging start.");
reload(context);
}
@Override
public void reset(LoggerContext context) {
log.info("LogStash logging reset.");
reload(context);
}
private void reload(LoggerContext context) {
MicaLoggingProperties.Loki loki = properties.getLoki();
if (loki.isEnabled()) {
addLokiAppender(context, loki);
}
}
private static void addLokiAppender(LoggerContext context,
MicaLoggingProperties.Loki properties) {
Loki4jAppender lokiAppender = new Loki4jAppender();
lokiAppender.setName(APPENDER_NAME);
// 通用配置
lokiAppender.setBatchMaxItems(properties.getBatchMaxItems());
lokiAppender.setBatchMaxBytes(properties.getBatchMaxBytes());
lokiAppender.setBatchTimeoutMs(properties.getBatchTimeoutMs());
lokiAppender.setSendQueueMaxBytes(properties.getSendQueueMaxBytes());
lokiAppender.setUseDirectBuffers(properties.isUseDirectBuffers());
lokiAppender.setDrainOnStop(properties.isDrainOnStop());
lokiAppender.setMetricsEnabled(properties.isMetricsEnabled());
lokiAppender.setVerbose(properties.isVerbose());
// format
lokiAppender.setFormat(getFormat(properties));
// http
lokiAppender.setHttp(getSender(properties));
// 先删除,再添加
context.getLogger(Logger.ROOT_LOGGER_NAME).detachAppender(APPENDER_NAME);
context.getLogger(Logger.ROOT_LOGGER_NAME).addAppender(lokiAppender);
}
private static Loki4jEncoder getFormat(MicaLoggingProperties.Loki properties) {
MicaLoggingProperties.LokiEncoder encoder = properties.getEncoder();
AbstractLoki4jEncoder loki4jEncoder = MicaLoggingProperties.LokiEncoder.ProtoBuf == encoder ?
new ProtobufEncoder() : new JsonEncoder();
// label config
AbstractLoki4jEncoder.LabelCfg labelCfg = new AbstractLoki4jEncoder.LabelCfg();
labelCfg.setPattern(properties.getFormatLabelPattern());
labelCfg.setPairSeparator(properties.getFormatLabelPairSeparator());
labelCfg.setKeyValueSeparator(properties.getFormatLabelKeyValueSeparator());
labelCfg.setNopex(properties.isFormatLabelNoPex());
loki4jEncoder.setLabel(labelCfg);
// message config
AbstractLoki4jEncoder.MessageCfg messageCfg = new AbstractLoki4jEncoder.MessageCfg();
messageCfg.setPattern(properties.getFormatMessagePattern());
loki4jEncoder.setMessage(messageCfg);
// 其他配置
loki4jEncoder.setStaticLabels(properties.isFormatStaticLabels());
loki4jEncoder.setSortByTime(properties.isFormatSortByTime());
return loki4jEncoder;
}
private static HttpSender getSender(MicaLoggingProperties.Loki properties) {
MicaLoggingProperties.HttpSender httpSender = properties.getHttpSender();
AbstractHttpSender abstractHttpSender;
if (MicaLoggingProperties.HttpSender.OK_HTTP == httpSender) {
abstractHttpSender = new OkHttpSender();
} else if (MicaLoggingProperties.HttpSender.APACHE_HTTP == httpSender) {
abstractHttpSender = new ApacheHttpSender();
} else {
abstractHttpSender = new JavaHttpSender();
}
abstractHttpSender.setUrl(properties.getHttpUrl());
abstractHttpSender.setConnectionTimeoutMs(properties.getHttpConnectionTimeoutMs());
abstractHttpSender.setRequestTimeoutMs(properties.getHttpRequestTimeoutMs());
String authUsername = properties.getHttpAuthUsername();
String authPassword = properties.getHttpAuthPassword();
if (StringUtil.isNotBlank(authUsername) && StringUtil.isNotBlank(authPassword)) {
AbstractHttpSender.BasicAuth basicAuth = new AbstractHttpSender.BasicAuth();
basicAuth.setUsername(authUsername);
basicAuth.setPassword(authPassword);
abstractHttpSender.setAuth(basicAuth);
}
abstractHttpSender.setTenantId(properties.getHttpAuthTenantId());
return abstractHttpSender;
}
}
......@@ -96,6 +96,17 @@ public class MicaLoggingConfiguration {
}
}
@Configuration(proxyBeanMethods = false)
@Conditional(LoggingCondition.class)
@ConditionalOnAppender(Appender.LOKI)
public static class LoggingLokiConfiguration {
@Bean
public LoggingLokiAppender loggingLokiAppender(MicaLoggingProperties properties) {
return new LoggingLokiAppender(properties);
}
}
@Target({ElementType.TYPE, ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
@Documented
......@@ -114,6 +125,7 @@ public class MicaLoggingConfiguration {
@Order(Ordered.HIGHEST_PRECEDENCE)
private static class LoggingCondition extends SpringBootCondition {
private static final String LOG_STASH_CLASS_NAME = "net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder";
private static final String Loki_CLASS_NAME = "com.github.loki4j.logback.Loki4jAppender";
@Override
public ConditionOutcome getMatchOutcome(ConditionContext context, AnnotatedTypeMetadata metadata) {
......@@ -124,7 +136,16 @@ public class MicaLoggingConfiguration {
ClassLoader classLoader = context.getClassLoader();
Boolean fileEnabled = environment.getProperty(MicaLoggingProperties.Files.PREFIX + ".enabled", Boolean.class, Boolean.TRUE);
Boolean logStashEnabled = environment.getProperty(MicaLoggingProperties.Logstash.PREFIX + ".enabled", Boolean.class, Boolean.FALSE);
if (Appender.LOG_STASH == appender) {
Boolean lokiEnabled = environment.getProperty(MicaLoggingProperties.Loki.PREFIX + ".enabled", Boolean.class, Boolean.FALSE);
if (Appender.LOKI == appender) {
if (ObjectUtil.isFalse(lokiEnabled)) {
return ConditionOutcome.noMatch("Logging loki is not enabled.");
}
if (hasLokiDependencies(classLoader)) {
return ConditionOutcome.match();
}
throw new IllegalStateException("Logging loki is enabled, please add com.github.loki4j loki-logback-appender dependencies.");
} else if (Appender.LOG_STASH == appender) {
if (ObjectUtil.isFalse(logStashEnabled)) {
return ConditionOutcome.noMatch("Logging logstash is not enabled.");
}
......@@ -155,6 +176,10 @@ public class MicaLoggingConfiguration {
private static boolean hasLogStashDependencies(ClassLoader classLoader) {
return ClassUtils.isPresent(LOG_STASH_CLASS_NAME, classLoader);
}
private static boolean hasLokiDependencies(ClassLoader classLoader) {
return ClassUtils.isPresent(Loki_CLASS_NAME, classLoader);
}
}
}
......@@ -36,6 +36,7 @@ public class MicaLoggingProperties {
private final Console console = new Console();
private final Files files = new Files();
private final Logstash logstash = new Logstash();
private final Loki loki = new Loki();
@Getter
@Setter
......@@ -69,16 +70,90 @@ public class MicaLoggingProperties {
*/
private boolean enabled = false;
/**
* logstash host
* 目标地址,默认: localhost:5000,示例: host1.domain.com,host2.domain.com:5560
*/
private String host = "localhost";
/**
* logstash port
*/
private int port = 5000;
private String destinations = "localhost:5000";
/**
* logstash 队列大小
*/
private int queueSize = 512;
}
@Getter
@Setter
public static class Loki {
public static final String PREFIX = MicaLoggingProperties.PREFIX + ".loki";
/**
* 是否开启 loki 日志收集
*/
private boolean enabled = false;
/**
* 编码方式
*/
private LokiEncoder encoder = LokiEncoder.Json;
/**
* http sender,默认: java11
*/
private HttpSender httpSender = HttpSender.JAVA_11;
/**
* 通用配置
*/
private int batchMaxItems = 1000;
private int batchMaxBytes = 4 * 1024 * 1024;
private long batchTimeoutMs = 60000;
private long sendQueueMaxBytes = 41943040;
/**
* 使用堆外内存
*/
private boolean useDirectBuffers = true;
private boolean drainOnStop = true;
/**
* 开启 metrics
*/
private boolean metricsEnabled = false;
private boolean verbose = false;
/**
* http 配置
*/
private String httpUrl;
private long httpConnectionTimeoutMs = 30000;
private long httpRequestTimeoutMs = 5000;
private String httpAuthUsername;
private String httpAuthPassword;
private String httpAuthTenantId;
/**
* format 配置
*/
private String formatLabelPattern;
private String formatLabelPairSeparator = ",";
private String formatLabelKeyValueSeparator = "=";
private boolean formatLabelNoPex = true;
private String formatMessagePattern;
private boolean formatStaticLabels = false;
private boolean formatSortByTime = false;
}
/**
* 编码方式
*/
public enum LokiEncoder {
/**
* Encoder
*/
Json,
ProtoBuf
}
/**
* http Sender
*/
public enum HttpSender {
/**
* http 方式
*/
JAVA_11,
OK_HTTP,
APACHE_HTTP
}
}
/*
* Copyright (c) 2019-2029, Dreamlu 卢春梦 (596392912@qq.com & www.dreamlu.net).
* <p>
* Licensed under the GNU LESSER GENERAL PUBLIC LICENSE 3.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.gnu.org/licenses/lgpl.html
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package net.dreamlu.mica.logging.loki;
import ch.qos.logback.core.joran.spi.NoAutoStart;
import com.github.loki4j.common.HttpHeaders;
import com.github.loki4j.common.LokiResponse;
import com.github.loki4j.logback.AbstractHttpSender;
import okhttp3.*;
import okhttp3.internal.Util;
import java.io.IOException;
import java.nio.ByteBuffer;
/**
* Loki sender that is backed by OkHttp
*
* @author L.cm
*/
@NoAutoStart
public class OkHttpSender extends AbstractHttpSender {
private OkHttpClient httpClient;
private MediaType mediaType;
/**
* Buffer is needed for turning ByteBuffer into byte array
* only if direct ByteBuffer arrived.
*/
private byte[] bodyBuffer = new byte[0];
@Override
public void start() {
super.start();
httpClient = new OkHttpClient();
mediaType = MediaType.get(contentType);
}
@Override
public void stop() {
super.stop();
httpClient.dispatcher().executorService().shutdown();
httpClient.connectionPool().evictAll();
Util.closeQuietly(httpClient.cache());
}
@Override
public LokiResponse send(ByteBuffer batch) {
Request.Builder request = new Request.Builder()
.url(getUrl())
.addHeader(HttpHeaders.CONTENT_TYPE, contentType);
tenantId.ifPresent(tenant -> request.addHeader(HttpHeaders.X_SCOPE_ORGID, tenant));
basicAuthToken.ifPresent(token -> request.addHeader(HttpHeaders.AUTHORIZATION, "Basic " + token));
if (batch.hasArray()) {
request.post(RequestBody.create(mediaType, batch.array(), batch.position(), batch.remaining()));
} else {
int len = batch.remaining();
if (len > bodyBuffer.length) {
bodyBuffer = new byte[len];
}
batch.get(bodyBuffer, 0, len);
request.post(RequestBody.create(mediaType, bodyBuffer, 0, len));
}
Call call = httpClient.newCall(request.build());
try (Response response = call.execute()) {
String body = response.body() != null ? response.body().string() : "";
return new LokiResponse(response.code(), body);
} catch (IOException e) {
throw new RuntimeException("Error while sending batch to Loki", e);
}
}
}
<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true">
<include resource="org/springframework/boot/logging/logback/defaults.xml" />
<!-- 控制台日志 -->
<include resource="org/springframework/boot/logging/logback/console-appender.xml" />
<root level="${rootLogLevel}">
<appender-ref ref="CONSOLE" />
</root>
<!-- 减少部分debug日志 -->
<logger name="org.springframework.context" level="WARN"/>
<logger name="org.springframework.beans" level="WARN"/>
<logger name="springfox.bean.validators" level="ERROR"/>
<logger name="springfox.documentation" level="ERROR" />
<!-- 关闭 mybatis 默认的 sql 日志 -->
<logger name="log.mybatis" level="INFO"/>
<!-- 基础组件 -->
<logger name="RocketmqClient" level="WARN"/>
<logger name="com.alibaba.nacos" level="ERROR"/>
<Logger name="org.reflections" level="ERROR"/>
<!-- 请求日志打印,全部使用info,方便动态调整 -->
<logger name="net.dreamlu.mica.servlet.logger" level="INFO"/>
<logger name="net.dreamlu.mica.reactive.logger" level="INFO"/>
<logger name="net.dreamlu.mica.http.logger" level="INFO"/>
<!-- https://logback.qos.ch/manual/configuration.html#shutdownHook and https://jira.qos.ch/browse/LOGBACK-1090 -->
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
<resetJUL>true</resetJUL>
</contextListener>
</configuration>
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册