feat: db connection config

.gitignore

@@ -2,3 +2,4 @@
 *.iml
 target
 logs
+askdata.log

script/restart.sh

+#!/bin/bash
+
+export AD_DB_HOST=127.0.0.1
+export AD_DB_PORT=3306
+export AD_DB_NAME=askdata_dev
+export AD_DB_USER=askdata_user
+export AD_DB_PASS=askdata624@MySQL
+
+# mvn -DskipTests clean package
+ps -ef | grep 'askdata-backend' | grep -v grep | awk '{print $2}' | xargs kill -9
+rm -frv askdata.log
+nohup java -jar target/askdata-backend-1.0-SNAPSHOT.jar >> askdata.log 2>&1 &
+sleep 1
+tail -fn 200 askdata.log
\ No newline at end of file

src/main/java/org/enthusa/askdata/common/Consts.java

+package org.enthusa.askdata.common;
+
+/**
+ * @author henry
+ * @date 2023/7/8
+ */
+public interface Consts {
+    String MYSQL_JDBC_URL_DECORATOR = "useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=GMT%2b8";
+}

src/main/java/org/enthusa/askdata/config/GlobalSetting.java

@@ -21,4 +21,19 @@ public class GlobalSetting {
 
     @Value("${settings.openai.key}")
     private String openAiKey;
+
+    @Value("${settings.mysql.host}")
+    private String dbHost;
+
+    @Value("${settings.mysql.port}")
+    private String dbPort;
+
+    @Value("${settings.mysql.name}")
+    private String dbName;
+
+    @Value("${settings.mysql.user}")
+    private String dbUsername;
+
+    @Value("${settings.mysql.pass}")
+    private String dbPassword;
 }

src/main/java/org/enthusa/askdata/config/MysqlAskDataConfig.java

+package org.enthusa.askdata.config;
+
+import com.zaxxer.hikari.HikariDataSource;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.enthusa.askdata.common.Consts;
+import org.enthusa.avatar.mybatis.utils.DbUtil;
+import org.mybatis.spring.SqlSessionFactoryBean;
+import org.mybatis.spring.annotation.MapperScan;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.Resource;
+import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
+import org.springframework.core.io.support.ResourcePatternResolver;
+
+import javax.sql.DataSource;
+import java.io.FileNotFoundException;
+
+/**
+ * @author henry
+ * @date 2023/2/18
+ */
+@Slf4j
+@Configuration
+@MapperScan(basePackages = "org.enthusa.askdata.mapper", sqlSessionFactoryRef = "askDataSqlSessionFactory")
+public class MysqlAskDataConfig {
+    private static final String MAPPER_LOCATION = "classpath*:org/enthusa/askdata/mapper/*Mapper.xml";
+
+    @Autowired
+    private GlobalSetting globalSetting;
+
+    @Bean
+    @Primary
+    public DataSource askDataDataSource() {
+        String dbHost = StringUtils.defaultIfBlank(System.getenv("AD_DB_HOST"), globalSetting.getDbHost());
+        String dbPort = StringUtils.defaultIfBlank(System.getenv("AD_DB_PORT"), globalSetting.getDbPort());
+        String dbName = StringUtils.defaultIfBlank(System.getenv("AD_DB_NAME"), globalSetting.getDbName());
+        String username = StringUtils.defaultIfBlank(System.getenv("AD_DB_USER"), globalSetting.getDbUsername());
+        String password = StringUtils.defaultIfBlank(System.getenv("AD_DB_PASS"), globalSetting.getDbPassword());
+
+        HikariDataSource dataSource = new HikariDataSource();
+        dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver");
+        String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?%s", dbHost, dbPort, dbName, Consts.MYSQL_JDBC_URL_DECORATOR);
+        dataSource.setJdbcUrl(jdbcUrl);
+        dataSource.setUsername(username);
+        dataSource.setPassword(password);
+        return dataSource;
+    }
+
+    @Bean
+    @Primary
+    public SqlSessionFactory askDataSqlSessionFactory(@Qualifier("askDataDataSource") DataSource dataSource) throws Exception {
+        SqlSessionFactoryBean sessionFactoryBean = new SqlSessionFactoryBean();
+        sessionFactoryBean.setConfiguration(DbUtil.getMyBatisConfig());
+        sessionFactoryBean.setDataSource(dataSource);
+        ResourcePatternResolver resourceResolver = new PathMatchingResourcePatternResolver();
+        Resource[] resources = null;
+        try {
+            resources = resourceResolver.getResources(MAPPER_LOCATION);
+        } catch (FileNotFoundException e) {
+            log.info("no mapper file found in mapperLocation:{}", MAPPER_LOCATION);
+        }
+        if (resources != null) {
+            sessionFactoryBean.setMapperLocations(resources);
+        }
+        return sessionFactoryBean.getObject();
+    }
+}

src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java

 package org.enthusa.askdata.controller;
 
-import com.alibaba.fastjson.JSON;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
 import org.enthusa.askdata.entity.BiDataSource;
 import org.enthusa.askdata.mapper.BiDataSourceMapper;
 import org.enthusa.askdata.task.impl.FillMetaDataTask;
-import org.enthusa.avatar.core.consts.TextConstant;
 import org.enthusa.avatar.db.metadata.MetaDataUtils;
 import org.enthusa.avatar.face.type.PageModel;
 import org.enthusa.avatar.face.type.Result;
@@ -23,7 +20,7 @@ import java.io.IOException;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.List;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -39,7 +36,7 @@ public class BiDataSourceController {
     @GetMapping("/catalogs")
     public Result getAllCatalogs() {
         List<String> catalogs = biDataSourceMapper.selectAll().stream().flatMap(source -> {
-            fillCatalogList(source);
+            source.fillDerivedFieldsFromDatabase();
             return source.getCatalogList().stream();
         }).collect(Collectors.toList());
         return ResultUtil.success(catalogs);
@@ -61,14 +58,11 @@ public class BiDataSourceController {
     public Result show(Model model, @PathVariable("id") Integer id) throws IOException {
         Validate.idValid("id", id);
         BiDataSource dataSource = biDataSourceMapper.selectByPrimaryKey(id);
-        fillCatalogList(dataSource);
-        fillDetailObject(dataSource);
         Validate.hasRecord("id", id, dataSource);
+        dataSource.fillDerivedFieldsFromDatabase();
         model.addAttribute("dataSource", dataSource);
 
-        byte[] bytes = Base64.getDecoder().decode(dataSource.getDetails());
-        Properties info = JSON.parseObject(new String(bytes), Properties.class);
-        try (Connection conn = DriverManager.getConnection(info.getProperty("url"), info)) {
+        try (Connection conn = DriverManager.getConnection(dataSource.getJdbcUrl(), dataSource.getUser(), dataSource.getPassword())) {
             List<String> catalogs = MetaDataUtils.getCatalogs(conn.getMetaData());
             model.addAttribute("catalogs", catalogs);
         } catch (SQLException e) {
@@ -85,10 +79,9 @@ public class BiDataSourceController {
 
         BiDataSource dataSource = new BiDataSource();
         BeanUtils.copyProperties(biDataSource, dataSource);
-        fillCatalogs(dataSource);
-        fillDetails(dataSource);
-        fillMetaDataTask.start();
+        dataSource.convertToDatabaseValue();
         biDataSourceMapper.insertSelective(dataSource);
+        fillMetaDataTask.start();
         return ResultUtil.success(dataSource);
     }
 
@@ -103,39 +96,9 @@ public class BiDataSourceController {
         Validate.hasRecord("id", id, dataSource);
 
         BeanUtils.copyProperties(biDataSource, dataSource);
-        fillCatalogs(dataSource);
-        fillDetails(dataSource);
-        fillMetaDataTask.start();
+        dataSource.convertToDatabaseValue();
         biDataSourceMapper.updateByPrimaryKeySelective(dataSource);
+        fillMetaDataTask.start();
         return ResultUtil.success(dataSource);
     }
-
-    private void fillCatalogList(BiDataSource dataSource) {
-        dataSource.setCatalogList(TextConstant.COMMA_SPLITTER.splitToList(StringUtils.defaultString(dataSource.getCatalogs())));
-    }
-
-    private void fillCatalogs(BiDataSource dataSource) {
-        dataSource.setCatalogs(TextConstant.COMMA_JOINER.join(Optional.ofNullable(dataSource.getCatalogList()).orElse(Collections.emptyList())));
-    }
-
-    private void fillDetailObject(BiDataSource dataSource) {
-        byte[] bytes = Base64.getDecoder().decode(dataSource.getDetails());
-        Properties info = JSON.parseObject(new String(bytes), Properties.class);
-        dataSource.setUrl(info.getProperty("url"));
-        dataSource.setUser(info.getProperty("user"));
-        dataSource.setPassword(info.getProperty("password"));
-    }
-
-    private void fillDetails(BiDataSource dataSource) {
-        if (StringUtils.isAnyBlank(dataSource.getUrl(), dataSource.getUser(), dataSource.getPassword())) {
-            return;
-        }
-        Properties config = new Properties();
-        config.setProperty("url", dataSource.getUrl().trim());
-        config.setProperty("user", dataSource.getUser().trim());
-        config.setProperty("password", dataSource.getPassword().trim());
-        String text = JSON.toJSONString(config);
-        String details = Base64.getEncoder().encodeToString(text.getBytes());
-        dataSource.setDetails(details);
-    }
 }

src/main/java/org/enthusa/askdata/controller/BiVenusController.java

 package org.enthusa.askdata.controller;
 
 import com.alibaba.druid.sql.SQLUtils;
-import com.alibaba.fastjson.JSON;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.dbutils.QueryRunner;
@@ -101,9 +100,8 @@ public class BiVenusController {
 
     private Connection getConnection(Integer dsId) throws SQLException {
         BiDataSource ds = biDataSourceMapper.selectByPrimaryKey(dsId);
-        byte[] bytes = Base64.getDecoder().decode(ds.getDetails());
-        Properties info = JSON.parseObject(new String(bytes), Properties.class);
-        return DriverManager.getConnection(info.getProperty("url"), info);
+        ds.fillDerivedFieldsFromDatabase();
+        return DriverManager.getConnection(ds.getJdbcUrl(), ds.getUser(), ds.getPassword());
 
     }
 

src/main/java/org/enthusa/askdata/entity/BiDataSource.java

 package org.enthusa.askdata.entity;
 
+import com.alibaba.fastjson.JSON;
 import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
+import org.enthusa.askdata.common.Consts;
+import org.enthusa.avatar.core.consts.TextConstant;
 import org.enthusa.avatar.mybatis.annotation.Transient;
 
-import java.util.Date;
-import java.util.List;
+import java.util.*;
 
 @Data
 public class BiDataSource {
@@ -24,7 +27,13 @@ public class BiDataSource {
     private String details;
 
     @Transient
-    private String url;
+    private String dbHost;
+    @Transient
+    private String dbPort;
+    @Transient
+    private String dbName;
+    @Transient
+    private String jdbcUrl;
     @Transient
     private String user;
     @Transient
@@ -59,4 +68,30 @@ public class BiDataSource {
     public void setCatalogs(String catalogs) {
         this.catalogs = catalogs == null ? null : catalogs.trim();
     }
+
+    public void fillDerivedFieldsFromDatabase() {
+        catalogList = TextConstant.COMMA_SPLITTER.splitToList(StringUtils.defaultString(catalogs));
+
+        byte[] bytes = Base64.getDecoder().decode(details);
+        Properties info = JSON.parseObject(new String(bytes), Properties.class);
+        dbHost = info.getProperty("dbHost");
+        dbPort = info.getProperty("dbPort");
+        dbName = info.getProperty("dbName");
+        jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?%s", dbHost, dbPort, dbName, Consts.MYSQL_JDBC_URL_DECORATOR);
+        user = info.getProperty("user");
+        password = info.getProperty("password");
+    }
+
+    public void convertToDatabaseValue() {
+        catalogs = TextConstant.COMMA_JOINER.join(Optional.ofNullable(catalogList).orElse(Collections.emptyList()));
+
+        Properties config = new Properties();
+        config.setProperty("dbHost", dbHost.trim());
+        config.setProperty("dbPort", dbPort.trim());
+        config.setProperty("dbName", dbName.trim());
+        config.setProperty("user", user.trim());
+        config.setProperty("password", password.trim());
+        String text = JSON.toJSONString(config);
+        details = Base64.getEncoder().encodeToString(text.getBytes());
+    }
 }

src/main/java/org/enthusa/askdata/task/impl/FillMetaDataTask.java

 package org.enthusa.askdata.task.impl;
 
-import com.alibaba.fastjson.JSON;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.enthusa.askdata.dao.BiFieldDao;
@@ -12,7 +11,6 @@ import org.enthusa.askdata.mapper.BiDataSourceMapper;
 import org.enthusa.askdata.mapper.BiFieldMapper;
 import org.enthusa.askdata.mapper.BiTableMapper;
 import org.enthusa.askdata.task.AbstractTask;
-import org.enthusa.avatar.core.consts.TextConstant;
 import org.enthusa.avatar.db.metadata.ColumnEntity;
 import org.enthusa.avatar.db.metadata.MetaDataUtils;
 import org.enthusa.avatar.db.metadata.TableEntity;
@@ -24,7 +22,9 @@ import javax.annotation.Resource;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.DriverManager;
-import java.util.*;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
@@ -57,12 +57,10 @@ public class FillMetaDataTask extends AbstractTask {
         Set<Integer> biFieldsAfter = new HashSet<>();
         List<BiDataSource> dsList = biDataSourceMapper.selectAll();
         for (BiDataSource ds : dsList) {
-            byte[] bytes = Base64.getDecoder().decode(ds.getDetails());
-            Properties info = JSON.parseObject(new String(bytes), Properties.class);
-            List<String> catalogs = TextConstant.COMMA_SPLITTER.splitToList(ds.getCatalogs());
-            try (Connection conn = DriverManager.getConnection(info.getProperty("url"), info)) {
+            ds.fillDerivedFieldsFromDatabase();
+            try (Connection conn = DriverManager.getConnection(ds.getJdbcUrl(), ds.getUser(), ds.getPassword())) {
                 DatabaseMetaData metaData = conn.getMetaData();
-                for (String catalog : catalogs) {
+                for (String catalog : ds.getCatalogList()) {
                     List<TableEntity> tables = MetaDataUtils.getTables(metaData, catalog);
                     for (TableEntity table : tables) {
                         if (StringUtils.containsAny(table.getName(), "tmp", "temp", "bak", "backup")

src/main/resources/application-dev.yml

@@ -3,13 +3,11 @@ settings:
   openai:
     host: from_command_line
     key: from_command_line
-spring:
-  datasource:
-    driver-class-name: com.mysql.cj.jdbc.Driver
-    url: jdbc:mysql://127.0.0.1:3306/askdata_dev?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=GMT%2b8
-    username: askdata_user
-    password: askdata624@MySQL
-    schema: classpath:import.sql
-    initialization-mode: never
+  mysql:
+    host: 127.0.0.1
+    port: 3306
+    name: askdata_dev
+    user: askdata_user
+    pass: askdata624@MySQL
 logging:
   config: classpath:logback-dev.xml

src/main/resources/application-prod.yml

@@ -3,11 +3,11 @@ settings:
   openai:
     host: from_command_line
     key: from_command_line
-spring:
-  datasource:
-    driver-class-name: com.mysql.cj.jdbc.Driver
-    url: jdbc:mysql://mysql.inscode.run:3306/askdata_prod?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=GMT%2b8
-    username: root
-    password: inscode
+  mysql:
+    host: mysql.inscode.run
+    port: 3306
+    name: askdata_prod
+    user: root
+    pass: inscode
 logging:
   config: classpath:logback-prod.xml