diff --git a/lib/avatar-client-master-SNAPSHOT.jar b/lib/avatar-client-master-SNAPSHOT.jar
new file mode 100644
index 0000000000000000000000000000000000000000..8f6532defcf51db3e1ea8137dee9b5bed9a5e0c0
Binary files /dev/null and b/lib/avatar-client-master-SNAPSHOT.jar differ
diff --git a/lib/avatar-db-master-SNAPSHOT.jar b/lib/avatar-db-master-SNAPSHOT.jar
new file mode 100644
index 0000000000000000000000000000000000000000..bf4f4de1c1564da5e0505a89dfdb0c5f3dca44ed
Binary files /dev/null and b/lib/avatar-db-master-SNAPSHOT.jar differ
diff --git a/lib/avatar-utils-master-SNAPSHOT.jar b/lib/avatar-utils-master-SNAPSHOT.jar
new file mode 100644
index 0000000000000000000000000000000000000000..fd2d00e54ac2d8e057fe06b73c15c9533719e2e3
Binary files /dev/null and b/lib/avatar-utils-master-SNAPSHOT.jar differ
diff --git a/pom.xml b/pom.xml
index 316fa59522018881a9d70f88b00257c99420bf11..9f2ca2b627edb7e8d2dc80abec8f9ce0c06487fb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -50,6 +50,13 @@
system
${project.basedir}/lib/avatar-mybatis-master-SNAPSHOT.jar
+
+ org.enthusa.avatar
+ avatar-db
+ master-SNAPSHOT
+ system
+ ${project.basedir}/lib/avatar-db-master-SNAPSHOT.jar
+
org.enthusa.avatar
avatar-core
@@ -57,11 +64,30 @@
system
${project.basedir}/lib/avatar-core-master-SNAPSHOT.jar
+
+ org.enthusa.avatar
+ avatar-utils
+ master-SNAPSHOT
+ system
+ ${project.basedir}/lib/avatar-utils-master-SNAPSHOT.jar
+
+
+ org.enthusa.avatar
+ avatar-client
+ master-SNAPSHOT
+ system
+ ${project.basedir}/lib/avatar-client-master-SNAPSHOT.jar
+
org.projectlombok
lombok
+
+ com.alibaba
+ druid
+ 1.2.18
+
org.mybatis
mybatis
@@ -72,6 +98,29 @@
commons-text
1.9
+
+ commons-dbutils
+ commons-dbutils
+ 1.7
+
+
+ com.google.guava
+ guava
+ 30.0-jre
+
+
+ com.squareup.okhttp3
+ okhttp-sse
+
+
+ org.apache.httpcomponents
+ httpcore
+
+
+ com.alibaba
+ fastjson
+ 1.2.83
+
mysql
mysql-connector-java
diff --git a/src/main/java/org/enthusa/askdata/config/GlobalSetting.java b/src/main/java/org/enthusa/askdata/config/GlobalSetting.java
index c3444a0f6c8cac5a615ab3772e2f3af05f2542f7..749bea13f9c23c8cc01078b6db46cd7e6c307855 100644
--- a/src/main/java/org/enthusa/askdata/config/GlobalSetting.java
+++ b/src/main/java/org/enthusa/askdata/config/GlobalSetting.java
@@ -15,4 +15,10 @@ import org.springframework.stereotype.Component;
public class GlobalSetting {
@Value("${settings.env}")
private String env;
+
+ @Value("${settings.openai.host}")
+ private String openAiHost;
+
+ @Value("${settings.openai.key}")
+ private String openAiKey;
}
diff --git a/src/main/java/org/enthusa/askdata/config/WebConfig.java b/src/main/java/org/enthusa/askdata/config/WebConfig.java
new file mode 100644
index 0000000000000000000000000000000000000000..1edb3101236c553087c722743c7b7310103dc27f
--- /dev/null
+++ b/src/main/java/org/enthusa/askdata/config/WebConfig.java
@@ -0,0 +1,31 @@
+package org.enthusa.askdata.config;
+
+import org.enthusa.askdata.ext.inscode.GptClient;
+import org.enthusa.avatar.ext.openai.OpenAiClient;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import javax.annotation.Resource;
+
+/**
+ * @author henry
+ * @date 2023/7/1
+ */
+@Configuration
+public class WebConfig {
+ @Resource
+ private GlobalSetting globalSetting;
+
+ @Bean
+ public GptClient gptClient() {
+ return GptClient.builder().build();
+ }
+
+ @Bean
+ public OpenAiClient openAiClient() {
+ return OpenAiClient.builder()
+ .apiKey(globalSetting.getOpenAiKey())
+ .apiHost(globalSetting.getOpenAiHost())
+ .build();
+ }
+}
diff --git a/src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java b/src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java
new file mode 100644
index 0000000000000000000000000000000000000000..924dac99c11bc1cb3cf2f6944292b2afe9f67c5a
--- /dev/null
+++ b/src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java
@@ -0,0 +1,141 @@
+package org.enthusa.askdata.controller;
+
+import com.alibaba.fastjson.JSON;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.enthusa.askdata.entity.BiDataSource;
+import org.enthusa.askdata.mapper.BiDataSourceMapper;
+import org.enthusa.askdata.task.impl.FillMetaDataTask;
+import org.enthusa.avatar.core.consts.TextConstant;
+import org.enthusa.avatar.db.metadata.MetaDataUtils;
+import org.enthusa.avatar.face.type.PageModel;
+import org.enthusa.avatar.face.type.Result;
+import org.enthusa.avatar.face.utils.ResultUtil;
+import org.enthusa.avatar.face.utils.Validate;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.ui.Model;
+import org.springframework.validation.BindingResult;
+import org.springframework.web.bind.annotation.*;
+
+import javax.validation.Valid;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@RestController
+@RequestMapping("/api/bi")
+public class BiDataSourceController {
+ @Autowired
+ private BiDataSourceMapper biDataSourceMapper;
+
+ @Autowired
+ private FillMetaDataTask fillMetaDataTask;
+
+ @GetMapping("/catalogs")
+ public Result getAllCatalogs() {
+ List catalogs = biDataSourceMapper.selectAll().stream().flatMap(source -> {
+ fillCatalogList(source);
+ return source.getCatalogList().stream();
+ }).collect(Collectors.toList());
+ return ResultUtil.success(catalogs);
+ }
+
+ @GetMapping("/datasources")
+ public Result index(
+ @RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
+ @RequestParam(value = "pageSize", required = false, defaultValue = "20") Integer pageSize
+ ) {
+ PageModel pageModel = new PageModel<>();
+ pageModel.setPageAndPageSize(page, pageSize);
+ List dataSourceList = biDataSourceMapper.selectByPage(pageModel);
+ pageModel.setList(dataSourceList);
+ return ResultUtil.success(pageModel);
+ }
+
+ @GetMapping("/datasources/{id}")
+ public Result show(Model model, @PathVariable("id") Integer id) throws IOException {
+ Validate.idValid("id", id);
+ BiDataSource dataSource = biDataSourceMapper.selectByPrimaryKey(id);
+ fillCatalogList(dataSource);
+ fillDetailObject(dataSource);
+ Validate.hasRecord("id", id, dataSource);
+ model.addAttribute("dataSource", dataSource);
+
+ byte[] bytes = Base64.getDecoder().decode(dataSource.getDetails());
+ Properties info = JSON.parseObject(new String(bytes), Properties.class);
+ try (Connection conn = DriverManager.getConnection(info.getProperty("url"), info)) {
+ List catalogs = MetaDataUtils.getCatalogs(conn.getMetaData());
+ model.addAttribute("catalogs", catalogs);
+ } catch (SQLException e) {
+ log.error(e.getMessage(), e);
+ }
+ return ResultUtil.success(model);
+ }
+
+ @PostMapping("/datasources")
+ public Result create(@RequestBody @Valid BiDataSource biDataSource, BindingResult bindingResult) {
+ if (bindingResult.hasErrors()) {
+ Validate.isRecord(true, bindingResult.getFieldError().getDefaultMessage());
+ }
+
+ BiDataSource dataSource = new BiDataSource();
+ BeanUtils.copyProperties(biDataSource, dataSource);
+ fillCatalogs(dataSource);
+ fillDetails(dataSource);
+ fillMetaDataTask.start();
+ biDataSourceMapper.insertSelective(dataSource);
+ return ResultUtil.success(dataSource);
+ }
+
+ @PutMapping("/datasources/{id}")
+ public Result update(@PathVariable("id") Integer id, @RequestBody @Valid BiDataSource biDataSource, BindingResult bindingResult) {
+ if (bindingResult.hasErrors()) {
+ Validate.isRecord(true, bindingResult.getFieldError().getDefaultMessage());
+ }
+
+ Validate.idValid("id", id);
+ BiDataSource dataSource = biDataSourceMapper.selectByPrimaryKey(id);
+ Validate.hasRecord("id", id, dataSource);
+
+ BeanUtils.copyProperties(biDataSource, dataSource);
+ fillCatalogs(dataSource);
+ fillDetails(dataSource);
+ fillMetaDataTask.start();
+ biDataSourceMapper.updateByPrimaryKeySelective(dataSource);
+ return ResultUtil.success(dataSource);
+ }
+
+ private void fillCatalogList(BiDataSource dataSource) {
+ dataSource.setCatalogList(TextConstant.COMMA_SPLITTER.splitToList(StringUtils.defaultString(dataSource.getCatalogs())));
+ }
+
+ private void fillCatalogs(BiDataSource dataSource) {
+ dataSource.setCatalogs(TextConstant.COMMA_JOINER.join(Optional.ofNullable(dataSource.getCatalogList()).orElse(Collections.emptyList())));
+ }
+
+ private void fillDetailObject(BiDataSource dataSource) {
+ byte[] bytes = Base64.getDecoder().decode(dataSource.getDetails());
+ Properties info = JSON.parseObject(new String(bytes), Properties.class);
+ dataSource.setUrl(info.getProperty("url"));
+ dataSource.setUser(info.getProperty("user"));
+ dataSource.setPassword(info.getProperty("password"));
+ }
+
+ private void fillDetails(BiDataSource dataSource) {
+ if (StringUtils.isAnyBlank(dataSource.getUrl(), dataSource.getUser(), dataSource.getPassword())) {
+ return;
+ }
+ Properties config = new Properties();
+ config.setProperty("url", dataSource.getUrl().trim());
+ config.setProperty("user", dataSource.getUser().trim());
+ config.setProperty("password", dataSource.getPassword().trim());
+ String text = JSON.toJSONString(config);
+ String details = Base64.getEncoder().encodeToString(text.getBytes());
+ dataSource.setDetails(details);
+ }
+}
diff --git a/src/main/java/org/enthusa/askdata/controller/BiPostController.java b/src/main/java/org/enthusa/askdata/controller/BiPostController.java
index ac4f12dfe114e6a33567459955569e9f59b64506..ccc0a45518bfcaca3cddbfbdb7d3c0a6b93ad2e1 100644
--- a/src/main/java/org/enthusa/askdata/controller/BiPostController.java
+++ b/src/main/java/org/enthusa/askdata/controller/BiPostController.java
@@ -15,7 +15,7 @@ import javax.validation.Valid;
import java.util.List;
@RestController
-@RequestMapping("/api")
+@RequestMapping("/api/bi")
public class BiPostController {
@Resource
private BiPostMapper biPostMapper;
diff --git a/src/main/java/org/enthusa/askdata/controller/BiTableController.java b/src/main/java/org/enthusa/askdata/controller/BiTableController.java
new file mode 100644
index 0000000000000000000000000000000000000000..5c5796637577b267c39aa681d114d4767c56531d
--- /dev/null
+++ b/src/main/java/org/enthusa/askdata/controller/BiTableController.java
@@ -0,0 +1,69 @@
+package org.enthusa.askdata.controller;
+
+import org.enthusa.askdata.entity.BiTable;
+import org.enthusa.askdata.mapper.BiTableMapper;
+import org.enthusa.avatar.face.type.PageModel;
+import org.enthusa.avatar.face.type.Result;
+import org.enthusa.avatar.face.utils.ResultUtil;
+import org.enthusa.avatar.face.utils.Validate;
+import org.springframework.beans.BeanUtils;
+import org.springframework.validation.BindingResult;
+import org.springframework.web.bind.annotation.*;
+
+import javax.annotation.Resource;
+import javax.validation.Valid;
+import java.util.List;
+
+@RestController
+@RequestMapping("/api/bi")
+public class BiTableController {
+ @Resource
+ private BiTableMapper biTableMapper;
+
+ @GetMapping("/tables")
+ public Result index(
+ @RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
+ @RequestParam(value = "pageSize", required = false, defaultValue = "20") Integer pageSize
+ ) {
+ PageModel pageModel = new PageModel<>();
+ pageModel.setPageAndPageSize(page, pageSize);
+ List tableList = biTableMapper.selectByPage(pageModel);
+ pageModel.setList(tableList);
+ return ResultUtil.success(pageModel);
+ }
+
+ @GetMapping("/tables/{id}")
+ public Result show(@PathVariable("id") Integer id) {
+ Validate.idValid("id", id);
+ BiTable table = biTableMapper.selectByPrimaryKey(id);
+ Validate.hasRecord("id", id, table);
+ return ResultUtil.success(table);
+ }
+
+ @PostMapping("/tables")
+ public Result create(@RequestBody @Valid BiTable biTable, BindingResult bindingResult) {
+ if (bindingResult.hasErrors()) {
+ Validate.isRecord(true, bindingResult.getFieldError().getDefaultMessage());
+ }
+
+ BiTable table = new BiTable();
+ BeanUtils.copyProperties(biTable, table);
+ biTableMapper.insertSelective(table);
+ return ResultUtil.success(table);
+ }
+
+ @PutMapping("/tables/{id}")
+ public Result update(@PathVariable("id") Integer id, @RequestBody @Valid BiTable biTable, BindingResult bindingResult) {
+ if (bindingResult.hasErrors()) {
+ Validate.isRecord(true, bindingResult.getFieldError().getDefaultMessage());
+ }
+
+ Validate.idValid("id", id);
+ BiTable table = biTableMapper.selectByPrimaryKey(id);
+ Validate.hasRecord("id", id, table);
+
+ BeanUtils.copyProperties(biTable, table);
+ biTableMapper.updateByPrimaryKey(table);
+ return ResultUtil.success(table);
+ }
+}
diff --git a/src/main/java/org/enthusa/askdata/controller/BiVenusController.java b/src/main/java/org/enthusa/askdata/controller/BiVenusController.java
new file mode 100644
index 0000000000000000000000000000000000000000..e6647f237352e7dd8b1b3c0e91fb95b698f1cc8a
--- /dev/null
+++ b/src/main/java/org/enthusa/askdata/controller/BiVenusController.java
@@ -0,0 +1,175 @@
+package org.enthusa.askdata.controller;
+
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.fastjson.JSON;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.MapListHandler;
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.enthusa.askdata.entity.BiDataSource;
+import org.enthusa.askdata.entity.BiField;
+import org.enthusa.askdata.entity.BiTable;
+import org.enthusa.askdata.ext.inscode.GptClient;
+import org.enthusa.askdata.ext.inscode.GptRequest;
+import org.enthusa.askdata.mapper.BiDataSourceMapper;
+import org.enthusa.askdata.mapper.BiFieldMapper;
+import org.enthusa.askdata.mapper.BiTableMapper;
+import org.enthusa.avatar.core.utils.RegexUtil;
+import org.enthusa.avatar.face.type.PageModel;
+import org.enthusa.avatar.face.type.Result;
+import org.enthusa.avatar.face.utils.ResultUtil;
+import org.enthusa.avatar.face.utils.Validate;
+import org.springframework.util.CollectionUtils;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.annotation.Resource;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * @author henry
+ * @date 2023/7/1
+ */
+@Slf4j
+@RestController
+@RequestMapping("/api/bi/venus")
+public class BiVenusController {
+ private static final Pattern LIMIT_PATTERN = Pattern.compile("limit\\s*\\d+[\\s\\d,;]*$");
+
+ @Resource
+ private BiDataSourceMapper biDataSourceMapper;
+
+ @Resource
+ private BiTableMapper biTableMapper;
+
+ @Resource
+ private BiFieldMapper biFieldMapper;
+
+ @Resource
+ private GptClient gptClient;
+
+ @PostMapping("/executeQuery")
+ public Result executeQuery(@RequestBody QueryVO queryVO) throws SQLException {
+ // Todo: 权限控制
+ if (StringUtils.containsAny(queryVO.getSql(), "pwd", "password", "email", "phone")) {
+ return ResultUtil.error(110, "包含敏感字段, 请修改 SQL 后再查询");
+ }
+
+ // Todo: 改写 SQL 还有很多细节, 比如分区检测
+ log.info("==> 改写前: {}", queryVO);
+ String sql = queryVO.getSql();
+ if (!RegexUtil.hasMatch(LIMIT_PATTERN, sql)) {
+ sql = sql.replaceFirst(";$", "") + " limit 1000";
+ }
+ log.info("==> 改写后: {}", sql);
+
+ // 数据源, 优先选数据表关联的数据源
+ Integer dsId = ObjectUtils.defaultIfNull(queryVO.getDs(), 1);
+ if (!CollectionUtils.isEmpty(queryVO.getTableIds())) {
+ Integer tableId = queryVO.getTableIds().get(0);
+ BiTable biTable = biTableMapper.selectByPrimaryKey(tableId);
+ dsId = biTable.getDsId();
+ }
+
+ QueryRunner qr = new QueryRunner();
+ Map model = new HashMap<>();
+ try (Connection conn = getConnection(dsId)) {
+ long begin = System.currentTimeMillis();
+ List