From 420b1d44d9b6a82a177c0343c62636e21916a7af Mon Sep 17 00:00:00 2001 From: Henry He Date: Sat, 8 Jul 2023 20:56:48 +0800 Subject: [PATCH] feat: db connection config --- .gitignore | 1 + script/restart.sh | 14 ++++ .../org/enthusa/askdata/common/Consts.java | 9 +++ .../enthusa/askdata/config/GlobalSetting.java | 15 ++++ .../askdata/config/MysqlAskDataConfig.java | 72 +++++++++++++++++++ .../controller/BiDataSourceController.java | 53 +++----------- .../askdata/controller/BiVenusController.java | 6 +- .../enthusa/askdata/entity/BiDataSource.java | 41 ++++++++++- .../askdata/task/impl/FillMetaDataTask.java | 14 ++-- src/main/resources/application-dev.yml | 14 ++-- src/main/resources/application-prod.yml | 12 ++-- 11 files changed, 177 insertions(+), 74 deletions(-) create mode 100755 script/restart.sh create mode 100644 src/main/java/org/enthusa/askdata/common/Consts.java create mode 100644 src/main/java/org/enthusa/askdata/config/MysqlAskDataConfig.java diff --git a/.gitignore b/.gitignore index 34d8a72..a4047c7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.iml target logs +askdata.log diff --git a/script/restart.sh b/script/restart.sh new file mode 100755 index 0000000..df11040 --- /dev/null +++ b/script/restart.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +export AD_DB_HOST=127.0.0.1 +export AD_DB_PORT=3306 +export AD_DB_NAME=askdata_dev +export AD_DB_USER=askdata_user +export AD_DB_PASS=askdata624@MySQL + +# mvn -DskipTests clean package +ps -ef | grep 'askdata-backend' | grep -v grep | awk '{print $2}' | xargs kill -9 +rm -frv askdata.log +nohup java -jar target/askdata-backend-1.0-SNAPSHOT.jar >> askdata.log 2>&1 & +sleep 1 +tail -fn 200 askdata.log \ No newline at end of file diff --git a/src/main/java/org/enthusa/askdata/common/Consts.java b/src/main/java/org/enthusa/askdata/common/Consts.java new file mode 100644 index 0000000..3885ff2 --- /dev/null +++ b/src/main/java/org/enthusa/askdata/common/Consts.java @@ -0,0 +1,9 @@ +package org.enthusa.askdata.common; + +/** + * @author henry + * @date 2023/7/8 + */ +public interface Consts { + String MYSQL_JDBC_URL_DECORATOR = "useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=GMT%2b8"; +} diff --git a/src/main/java/org/enthusa/askdata/config/GlobalSetting.java b/src/main/java/org/enthusa/askdata/config/GlobalSetting.java index 749bea1..20b12ca 100644 --- a/src/main/java/org/enthusa/askdata/config/GlobalSetting.java +++ b/src/main/java/org/enthusa/askdata/config/GlobalSetting.java @@ -21,4 +21,19 @@ public class GlobalSetting { @Value("${settings.openai.key}") private String openAiKey; + + @Value("${settings.mysql.host}") + private String dbHost; + + @Value("${settings.mysql.port}") + private String dbPort; + + @Value("${settings.mysql.name}") + private String dbName; + + @Value("${settings.mysql.user}") + private String dbUsername; + + @Value("${settings.mysql.pass}") + private String dbPassword; } diff --git a/src/main/java/org/enthusa/askdata/config/MysqlAskDataConfig.java b/src/main/java/org/enthusa/askdata/config/MysqlAskDataConfig.java new file mode 100644 index 0000000..18c8431 --- /dev/null +++ b/src/main/java/org/enthusa/askdata/config/MysqlAskDataConfig.java @@ -0,0 +1,72 @@ +package org.enthusa.askdata.config; + +import com.zaxxer.hikari.HikariDataSource; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.ibatis.session.SqlSessionFactory; +import org.enthusa.askdata.common.Consts; +import org.enthusa.avatar.mybatis.utils.DbUtil; +import org.mybatis.spring.SqlSessionFactoryBean; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.core.io.support.ResourcePatternResolver; + +import javax.sql.DataSource; +import java.io.FileNotFoundException; + +/** + * @author henry + * @date 2023/2/18 + */ +@Slf4j +@Configuration +@MapperScan(basePackages = "org.enthusa.askdata.mapper", sqlSessionFactoryRef = "askDataSqlSessionFactory") +public class MysqlAskDataConfig { + private static final String MAPPER_LOCATION = "classpath*:org/enthusa/askdata/mapper/*Mapper.xml"; + + @Autowired + private GlobalSetting globalSetting; + + @Bean + @Primary + public DataSource askDataDataSource() { + String dbHost = StringUtils.defaultIfBlank(System.getenv("AD_DB_HOST"), globalSetting.getDbHost()); + String dbPort = StringUtils.defaultIfBlank(System.getenv("AD_DB_PORT"), globalSetting.getDbPort()); + String dbName = StringUtils.defaultIfBlank(System.getenv("AD_DB_NAME"), globalSetting.getDbName()); + String username = StringUtils.defaultIfBlank(System.getenv("AD_DB_USER"), globalSetting.getDbUsername()); + String password = StringUtils.defaultIfBlank(System.getenv("AD_DB_PASS"), globalSetting.getDbPassword()); + + HikariDataSource dataSource = new HikariDataSource(); + dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver"); + String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?%s", dbHost, dbPort, dbName, Consts.MYSQL_JDBC_URL_DECORATOR); + dataSource.setJdbcUrl(jdbcUrl); + dataSource.setUsername(username); + dataSource.setPassword(password); + return dataSource; + } + + @Bean + @Primary + public SqlSessionFactory askDataSqlSessionFactory(@Qualifier("askDataDataSource") DataSource dataSource) throws Exception { + SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean(); + sessionFactoryBean.setConfiguration(DbUtil.getMyBatisConfig()); + sessionFactoryBean.setDataSource(dataSource); + ResourcePatternResolver resourceResolver = new PathMatchingResourcePatternResolver(); + Resource[] resources = null; + try { + resources = resourceResolver.getResources(MAPPER_LOCATION); + } catch (FileNotFoundException e) { + log.info("no mapper file found in mapperLocation:{}", MAPPER_LOCATION); + } + if (resources != null) { + sessionFactoryBean.setMapperLocations(resources); + } + return sessionFactoryBean.getObject(); + } +} diff --git a/src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java b/src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java index 924dac9..685b8a1 100644 --- a/src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java +++ b/src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java @@ -1,12 +1,9 @@ package org.enthusa.askdata.controller; -import com.alibaba.fastjson.JSON; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.enthusa.askdata.entity.BiDataSource; import org.enthusa.askdata.mapper.BiDataSourceMapper; import org.enthusa.askdata.task.impl.FillMetaDataTask; -import org.enthusa.avatar.core.consts.TextConstant; import org.enthusa.avatar.db.metadata.MetaDataUtils; import org.enthusa.avatar.face.type.PageModel; import org.enthusa.avatar.face.type.Result; @@ -23,7 +20,7 @@ import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.*; +import java.util.List; import java.util.stream.Collectors; @Slf4j @@ -39,7 +36,7 @@ public class BiDataSourceController { @GetMapping("/catalogs") public Result getAllCatalogs() { List catalogs = biDataSourceMapper.selectAll().stream().flatMap(source -> { - fillCatalogList(source); + source.fillDerivedFieldsFromDatabase(); return source.getCatalogList().stream(); }).collect(Collectors.toList()); return ResultUtil.success(catalogs); @@ -61,14 +58,11 @@ public class BiDataSourceController { public Result show(Model model, @PathVariable("id") Integer id) throws IOException { Validate.idValid("id", id); BiDataSource dataSource = biDataSourceMapper.selectByPrimaryKey(id); - fillCatalogList(dataSource); - fillDetailObject(dataSource); Validate.hasRecord("id", id, dataSource); + dataSource.fillDerivedFieldsFromDatabase(); model.addAttribute("dataSource", dataSource); - byte[] bytes = Base64.getDecoder().decode(dataSource.getDetails()); - Properties info = JSON.parseObject(new String(bytes), Properties.class); - try (Connection conn = DriverManager.getConnection(info.getProperty("url"), info)) { + try (Connection conn = DriverManager.getConnection(dataSource.getJdbcUrl(), dataSource.getUser(), dataSource.getPassword())) { List catalogs = MetaDataUtils.getCatalogs(conn.getMetaData()); model.addAttribute("catalogs", catalogs); } catch (SQLException e) { @@ -85,10 +79,9 @@ public class BiDataSourceController { BiDataSource dataSource = new BiDataSource(); BeanUtils.copyProperties(biDataSource, dataSource); - fillCatalogs(dataSource); - fillDetails(dataSource); - fillMetaDataTask.start(); + dataSource.convertToDatabaseValue(); biDataSourceMapper.insertSelective(dataSource); + fillMetaDataTask.start(); return ResultUtil.success(dataSource); } @@ -103,39 +96,9 @@ public class BiDataSourceController { Validate.hasRecord("id", id, dataSource); BeanUtils.copyProperties(biDataSource, dataSource); - fillCatalogs(dataSource); - fillDetails(dataSource); - fillMetaDataTask.start(); + dataSource.convertToDatabaseValue(); biDataSourceMapper.updateByPrimaryKeySelective(dataSource); + fillMetaDataTask.start(); return ResultUtil.success(dataSource); } - - private void fillCatalogList(BiDataSource dataSource) { - dataSource.setCatalogList(TextConstant.COMMA_SPLITTER.splitToList(StringUtils.defaultString(dataSource.getCatalogs()))); - } - - private void fillCatalogs(BiDataSource dataSource) { - dataSource.setCatalogs(TextConstant.COMMA_JOINER.join(Optional.ofNullable(dataSource.getCatalogList()).orElse(Collections.emptyList()))); - } - - private void fillDetailObject(BiDataSource dataSource) { - byte[] bytes = Base64.getDecoder().decode(dataSource.getDetails()); - Properties info = JSON.parseObject(new String(bytes), Properties.class); - dataSource.setUrl(info.getProperty("url")); - dataSource.setUser(info.getProperty("user")); - dataSource.setPassword(info.getProperty("password")); - } - - private void fillDetails(BiDataSource dataSource) { - if (StringUtils.isAnyBlank(dataSource.getUrl(), dataSource.getUser(), dataSource.getPassword())) { - return; - } - Properties config = new Properties(); - config.setProperty("url", dataSource.getUrl().trim()); - config.setProperty("user", dataSource.getUser().trim()); - config.setProperty("password", dataSource.getPassword().trim()); - String text = JSON.toJSONString(config); - String details = Base64.getEncoder().encodeToString(text.getBytes()); - dataSource.setDetails(details); - } } diff --git a/src/main/java/org/enthusa/askdata/controller/BiVenusController.java b/src/main/java/org/enthusa/askdata/controller/BiVenusController.java index a479c7d..3293a3c 100644 --- a/src/main/java/org/enthusa/askdata/controller/BiVenusController.java +++ b/src/main/java/org/enthusa/askdata/controller/BiVenusController.java @@ -1,7 +1,6 @@ package org.enthusa.askdata.controller; import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.fastjson.JSON; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.dbutils.QueryRunner; @@ -101,9 +100,8 @@ public class BiVenusController { private Connection getConnection(Integer dsId) throws SQLException { BiDataSource ds = biDataSourceMapper.selectByPrimaryKey(dsId); - byte[] bytes = Base64.getDecoder().decode(ds.getDetails()); - Properties info = JSON.parseObject(new String(bytes), Properties.class); - return DriverManager.getConnection(info.getProperty("url"), info); + ds.fillDerivedFieldsFromDatabase(); + return DriverManager.getConnection(ds.getJdbcUrl(), ds.getUser(), ds.getPassword()); } diff --git a/src/main/java/org/enthusa/askdata/entity/BiDataSource.java b/src/main/java/org/enthusa/askdata/entity/BiDataSource.java index 3822587..1a4741b 100644 --- a/src/main/java/org/enthusa/askdata/entity/BiDataSource.java +++ b/src/main/java/org/enthusa/askdata/entity/BiDataSource.java @@ -1,10 +1,13 @@ package org.enthusa.askdata.entity; +import com.alibaba.fastjson.JSON; import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.enthusa.askdata.common.Consts; +import org.enthusa.avatar.core.consts.TextConstant; import org.enthusa.avatar.mybatis.annotation.Transient; -import java.util.Date; -import java.util.List; +import java.util.*; @Data public class BiDataSource { @@ -24,7 +27,13 @@ public class BiDataSource { private String details; @Transient - private String url; + private String dbHost; + @Transient + private String dbPort; + @Transient + private String dbName; + @Transient + private String jdbcUrl; @Transient private String user; @Transient @@ -59,4 +68,30 @@ public class BiDataSource { public void setCatalogs(String catalogs) { this.catalogs = catalogs == null ? null : catalogs.trim(); } + + public void fillDerivedFieldsFromDatabase() { + catalogList = TextConstant.COMMA_SPLITTER.splitToList(StringUtils.defaultString(catalogs)); + + byte[] bytes = Base64.getDecoder().decode(details); + Properties info = JSON.parseObject(new String(bytes), Properties.class); + dbHost = info.getProperty("dbHost"); + dbPort = info.getProperty("dbPort"); + dbName = info.getProperty("dbName"); + jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?%s", dbHost, dbPort, dbName, Consts.MYSQL_JDBC_URL_DECORATOR); + user = info.getProperty("user"); + password = info.getProperty("password"); + } + + public void convertToDatabaseValue() { + catalogs = TextConstant.COMMA_JOINER.join(Optional.ofNullable(catalogList).orElse(Collections.emptyList())); + + Properties config = new Properties(); + config.setProperty("dbHost", dbHost.trim()); + config.setProperty("dbPort", dbPort.trim()); + config.setProperty("dbName", dbName.trim()); + config.setProperty("user", user.trim()); + config.setProperty("password", password.trim()); + String text = JSON.toJSONString(config); + details = Base64.getEncoder().encodeToString(text.getBytes()); + } } diff --git a/src/main/java/org/enthusa/askdata/task/impl/FillMetaDataTask.java b/src/main/java/org/enthusa/askdata/task/impl/FillMetaDataTask.java index 7983055..263c4e7 100644 --- a/src/main/java/org/enthusa/askdata/task/impl/FillMetaDataTask.java +++ b/src/main/java/org/enthusa/askdata/task/impl/FillMetaDataTask.java @@ -1,6 +1,5 @@ package org.enthusa.askdata.task.impl; -import com.alibaba.fastjson.JSON; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.enthusa.askdata.dao.BiFieldDao; @@ -12,7 +11,6 @@ import org.enthusa.askdata.mapper.BiDataSourceMapper; import org.enthusa.askdata.mapper.BiFieldMapper; import org.enthusa.askdata.mapper.BiTableMapper; import org.enthusa.askdata.task.AbstractTask; -import org.enthusa.avatar.core.consts.TextConstant; import org.enthusa.avatar.db.metadata.ColumnEntity; import org.enthusa.avatar.db.metadata.MetaDataUtils; import org.enthusa.avatar.db.metadata.TableEntity; @@ -24,7 +22,9 @@ import javax.annotation.Resource; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.DriverManager; -import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -57,12 +57,10 @@ public class FillMetaDataTask extends AbstractTask { Set biFieldsAfter = new HashSet<>(); List dsList = biDataSourceMapper.selectAll(); for (BiDataSource ds : dsList) { - byte[] bytes = Base64.getDecoder().decode(ds.getDetails()); - Properties info = JSON.parseObject(new String(bytes), Properties.class); - List catalogs = TextConstant.COMMA_SPLITTER.splitToList(ds.getCatalogs()); - try (Connection conn = DriverManager.getConnection(info.getProperty("url"), info)) { + ds.fillDerivedFieldsFromDatabase(); + try (Connection conn = DriverManager.getConnection(ds.getJdbcUrl(), ds.getUser(), ds.getPassword())) { DatabaseMetaData metaData = conn.getMetaData(); - for (String catalog : catalogs) { + for (String catalog : ds.getCatalogList()) { List tables = MetaDataUtils.getTables(metaData, catalog); for (TableEntity table : tables) { if (StringUtils.containsAny(table.getName(), "tmp", "temp", "bak", "backup") diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 5306917..8ff7bd6 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -3,13 +3,11 @@ settings: openai: host: from_command_line key: from_command_line -spring: - datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/askdata_dev?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=GMT%2b8 - username: askdata_user - password: askdata624@MySQL - schema: classpath:import.sql - initialization-mode: never + mysql: + host: 127.0.0.1 + port: 3306 + name: askdata_dev + user: askdata_user + pass: askdata624@MySQL logging: config: classpath:logback-dev.xml diff --git a/src/main/resources/application-prod.yml b/src/main/resources/application-prod.yml index ddd887b..57f95b8 100644 --- a/src/main/resources/application-prod.yml +++ b/src/main/resources/application-prod.yml @@ -3,11 +3,11 @@ settings: openai: host: from_command_line key: from_command_line -spring: - datasource: - driver-class-name: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://mysql.inscode.run:3306/askdata_prod?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=GMT%2b8 - username: root - password: inscode + mysql: + host: mysql.inscode.run + port: 3306 + name: askdata_prod + user: root + pass: inscode logging: config: classpath:logback-prod.xml -- GitLab