feat: ask data

pom.xml

@@ -50,6 +50,13 @@
       <scope>system</scope>
       <systemPath>${project.basedir}/lib/avatar-mybatis-master-SNAPSHOT.jar</systemPath>
     </dependency>
+    <dependency>
+      <groupId>org.enthusa.avatar</groupId>
+      <artifactId>avatar-db</artifactId>
+      <version>master-SNAPSHOT</version>
+      <scope>system</scope>
+      <systemPath>${project.basedir}/lib/avatar-db-master-SNAPSHOT.jar</systemPath>
+    </dependency>
     <dependency>
       <groupId>org.enthusa.avatar</groupId>
       <artifactId>avatar-core</artifactId>
@@ -57,11 +64,30 @@
       <scope>system</scope>
       <systemPath>${project.basedir}/lib/avatar-core-master-SNAPSHOT.jar</systemPath>
     </dependency>
+    <dependency>
+      <groupId>org.enthusa.avatar</groupId>
+      <artifactId>avatar-utils</artifactId>
+      <version>master-SNAPSHOT</version>
+      <scope>system</scope>
+      <systemPath>${project.basedir}/lib/avatar-utils-master-SNAPSHOT.jar</systemPath>
+    </dependency>
+    <dependency>
+      <groupId>org.enthusa.avatar</groupId>
+      <artifactId>avatar-client</artifactId>
+      <version>master-SNAPSHOT</version>
+      <scope>system</scope>
+      <systemPath>${project.basedir}/lib/avatar-client-master-SNAPSHOT.jar</systemPath>
+    </dependency>
 
     <dependency>
       <groupId>org.projectlombok</groupId>
       <artifactId>lombok</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.alibaba</groupId>
+      <artifactId>druid</artifactId>
+      <version>1.2.18</version>
+    </dependency>
     <dependency>
       <groupId>org.mybatis</groupId>
       <artifactId>mybatis</artifactId>
@@ -72,6 +98,29 @@
       <artifactId>commons-text</artifactId>
       <version>1.9</version>
     </dependency>
+    <dependency>
+      <groupId>commons-dbutils</groupId>
+      <artifactId>commons-dbutils</artifactId>
+      <version>1.7</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>30.0-jre</version>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp-sse</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.alibaba</groupId>
+      <artifactId>fastjson</artifactId>
+      <version>1.2.83</version>
+    </dependency>
     <dependency>
       <groupId>mysql</groupId>
       <artifactId>mysql-connector-java</artifactId>

src/main/java/org/enthusa/askdata/config/GlobalSetting.java

@@ -15,4 +15,10 @@ import org.springframework.stereotype.Component;
 public class GlobalSetting {
     @Value("${settings.env}")
     private String env;
+
+    @Value("${settings.openai.host}")
+    private String openAiHost;
+
+    @Value("${settings.openai.key}")
+    private String openAiKey;
 }

src/main/java/org/enthusa/askdata/config/WebConfig.java

+package org.enthusa.askdata.config;
+
+import org.enthusa.askdata.ext.inscode.GptClient;
+import org.enthusa.avatar.ext.openai.OpenAiClient;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import javax.annotation.Resource;
+
+/**
+ * @author henry
+ * @date 2023/7/1
+ */
+@Configuration
+public class WebConfig {
+    @Resource
+    private GlobalSetting globalSetting;
+
+    @Bean
+    public GptClient gptClient() {
+        return GptClient.builder().build();
+    }
+
+    @Bean
+    public OpenAiClient openAiClient() {
+        return OpenAiClient.builder()
+                .apiKey(globalSetting.getOpenAiKey())
+                .apiHost(globalSetting.getOpenAiHost())
+                .build();
+    }
+}

src/main/java/org/enthusa/askdata/controller/BiDataSourceController.java

+package org.enthusa.askdata.controller;
+
+import com.alibaba.fastjson.JSON;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.enthusa.askdata.entity.BiDataSource;
+import org.enthusa.askdata.mapper.BiDataSourceMapper;
+import org.enthusa.askdata.task.impl.FillMetaDataTask;
+import org.enthusa.avatar.core.consts.TextConstant;
+import org.enthusa.avatar.db.metadata.MetaDataUtils;
+import org.enthusa.avatar.face.type.PageModel;
+import org.enthusa.avatar.face.type.Result;
+import org.enthusa.avatar.face.utils.ResultUtil;
+import org.enthusa.avatar.face.utils.Validate;
+import org.springframework.beans.BeanUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.ui.Model;
+import org.springframework.validation.BindingResult;
+import org.springframework.web.bind.annotation.*;
+
+import javax.validation.Valid;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Slf4j
+@RestController
+@RequestMapping("/api/bi")
+public class BiDataSourceController {
+    @Autowired
+    private BiDataSourceMapper biDataSourceMapper;
+
+    @Autowired
+    private FillMetaDataTask fillMetaDataTask;
+
+    @GetMapping("/catalogs")
+    public Result getAllCatalogs() {
+        List<String> catalogs = biDataSourceMapper.selectAll().stream().flatMap(source -> {
+            fillCatalogList(source);
+            return source.getCatalogList().stream();
+        }).collect(Collectors.toList());
+        return ResultUtil.success(catalogs);
+    }
+
+    @GetMapping("/datasources")
+    public Result index(
+            @RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
+            @RequestParam(value = "pageSize", required = false, defaultValue = "20") Integer pageSize
+    ) {
+        PageModel<BiDataSource> pageModel = new PageModel<>();
+        pageModel.setPageAndPageSize(page, pageSize);
+        List<BiDataSource> dataSourceList = biDataSourceMapper.selectByPage(pageModel);
+        pageModel.setList(dataSourceList);
+        return ResultUtil.success(pageModel);
+    }
+
+    @GetMapping("/datasources/{id}")
+    public Result show(Model model, @PathVariable("id") Integer id) throws IOException {
+        Validate.idValid("id", id);
+        BiDataSource dataSource = biDataSourceMapper.selectByPrimaryKey(id);
+        fillCatalogList(dataSource);
+        fillDetailObject(dataSource);
+        Validate.hasRecord("id", id, dataSource);
+        model.addAttribute("dataSource", dataSource);
+
+        byte[] bytes = Base64.getDecoder().decode(dataSource.getDetails());
+        Properties info = JSON.parseObject(new String(bytes), Properties.class);
+        try (Connection conn = DriverManager.getConnection(info.getProperty("url"), info)) {
+            List<String> catalogs = MetaDataUtils.getCatalogs(conn.getMetaData());
+            model.addAttribute("catalogs", catalogs);
+        } catch (SQLException e) {
+            log.error(e.getMessage(), e);
+        }
+        return ResultUtil.success(model);
+    }
+
+    @PostMapping("/datasources")
+    public Result create(@RequestBody @Valid BiDataSource biDataSource, BindingResult bindingResult) {
+        if (bindingResult.hasErrors()) {
+            Validate.isRecord(true, bindingResult.getFieldError().getDefaultMessage());
+        }
+
+        BiDataSource dataSource = new BiDataSource();
+        BeanUtils.copyProperties(biDataSource, dataSource);
+        fillCatalogs(dataSource);
+        fillDetails(dataSource);
+        fillMetaDataTask.start();
+        biDataSourceMapper.insertSelective(dataSource);
+        return ResultUtil.success(dataSource);
+    }
+
+    @PutMapping("/datasources/{id}")
+    public Result update(@PathVariable("id") Integer id, @RequestBody @Valid BiDataSource biDataSource, BindingResult bindingResult) {
+        if (bindingResult.hasErrors()) {
+            Validate.isRecord(true, bindingResult.getFieldError().getDefaultMessage());
+        }
+
+        Validate.idValid("id", id);
+        BiDataSource dataSource = biDataSourceMapper.selectByPrimaryKey(id);
+        Validate.hasRecord("id", id, dataSource);
+
+        BeanUtils.copyProperties(biDataSource, dataSource);
+        fillCatalogs(dataSource);
+        fillDetails(dataSource);
+        fillMetaDataTask.start();
+        biDataSourceMapper.updateByPrimaryKeySelective(dataSource);
+        return ResultUtil.success(dataSource);
+    }
+
+    private void fillCatalogList(BiDataSource dataSource) {
+        dataSource.setCatalogList(TextConstant.COMMA_SPLITTER.splitToList(StringUtils.defaultString(dataSource.getCatalogs())));
+    }
+
+    private void fillCatalogs(BiDataSource dataSource) {
+        dataSource.setCatalogs(TextConstant.COMMA_JOINER.join(Optional.ofNullable(dataSource.getCatalogList()).orElse(Collections.emptyList())));
+    }
+
+    private void fillDetailObject(BiDataSource dataSource) {
+        byte[] bytes = Base64.getDecoder().decode(dataSource.getDetails());
+        Properties info = JSON.parseObject(new String(bytes), Properties.class);
+        dataSource.setUrl(info.getProperty("url"));
+        dataSource.setUser(info.getProperty("user"));
+        dataSource.setPassword(info.getProperty("password"));
+    }
+
+    private void fillDetails(BiDataSource dataSource) {
+        if (StringUtils.isAnyBlank(dataSource.getUrl(), dataSource.getUser(), dataSource.getPassword())) {
+            return;
+        }
+        Properties config = new Properties();
+        config.setProperty("url", dataSource.getUrl().trim());
+        config.setProperty("user", dataSource.getUser().trim());
+        config.setProperty("password", dataSource.getPassword().trim());
+        String text = JSON.toJSONString(config);
+        String details = Base64.getEncoder().encodeToString(text.getBytes());
+        dataSource.setDetails(details);
+    }
+}

src/main/java/org/enthusa/askdata/controller/BiPostController.java

@@ -15,7 +15,7 @@ import javax.validation.Valid;
 import java.util.List;
 
 @RestController
-@RequestMapping("/api")
+@RequestMapping("/api/bi")
 public class BiPostController {
     @Resource
     private BiPostMapper biPostMapper;

src/main/java/org/enthusa/askdata/controller/BiTableController.java

+package org.enthusa.askdata.controller;
+
+import org.enthusa.askdata.entity.BiTable;
+import org.enthusa.askdata.mapper.BiTableMapper;
+import org.enthusa.avatar.face.type.PageModel;
+import org.enthusa.avatar.face.type.Result;
+import org.enthusa.avatar.face.utils.ResultUtil;
+import org.enthusa.avatar.face.utils.Validate;
+import org.springframework.beans.BeanUtils;
+import org.springframework.validation.BindingResult;
+import org.springframework.web.bind.annotation.*;
+
+import javax.annotation.Resource;
+import javax.validation.Valid;
+import java.util.List;
+
+@RestController
+@RequestMapping("/api/bi")
+public class BiTableController {
+    @Resource
+    private BiTableMapper biTableMapper;
+
+    @GetMapping("/tables")
+    public Result index(
+            @RequestParam(value = "page", required = false, defaultValue = "1") Integer page,
+            @RequestParam(value = "pageSize", required = false, defaultValue = "20") Integer pageSize
+    ) {
+        PageModel<BiTable> pageModel = new PageModel<>();
+        pageModel.setPageAndPageSize(page, pageSize);
+        List<BiTable> tableList = biTableMapper.selectByPage(pageModel);
+        pageModel.setList(tableList);
+        return ResultUtil.success(pageModel);
+    }
+
+    @GetMapping("/tables/{id}")
+    public Result show(@PathVariable("id") Integer id) {
+        Validate.idValid("id", id);
+        BiTable table = biTableMapper.selectByPrimaryKey(id);
+        Validate.hasRecord("id", id, table);
+        return ResultUtil.success(table);
+    }
+
+    @PostMapping("/tables")
+    public Result create(@RequestBody @Valid BiTable biTable, BindingResult bindingResult) {
+        if (bindingResult.hasErrors()) {
+            Validate.isRecord(true, bindingResult.getFieldError().getDefaultMessage());
+        }
+
+        BiTable table = new BiTable();
+        BeanUtils.copyProperties(biTable, table);
+        biTableMapper.insertSelective(table);
+        return ResultUtil.success(table);
+    }
+
+    @PutMapping("/tables/{id}")
+    public Result update(@PathVariable("id") Integer id, @RequestBody @Valid BiTable biTable, BindingResult bindingResult) {
+        if (bindingResult.hasErrors()) {
+            Validate.isRecord(true, bindingResult.getFieldError().getDefaultMessage());
+        }
+
+        Validate.idValid("id", id);
+        BiTable table = biTableMapper.selectByPrimaryKey(id);
+        Validate.hasRecord("id", id, table);
+
+        BeanUtils.copyProperties(biTable, table);
+        biTableMapper.updateByPrimaryKey(table);
+        return ResultUtil.success(table);
+    }
+}

src/main/java/org/enthusa/askdata/controller/BiVenusController.java

+package org.enthusa.askdata.controller;
+
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.fastjson.JSON;
+import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.MapListHandler;
+import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.enthusa.askdata.entity.BiDataSource;
+import org.enthusa.askdata.entity.BiField;
+import org.enthusa.askdata.entity.BiTable;
+import org.enthusa.askdata.ext.inscode.GptClient;
+import org.enthusa.askdata.ext.inscode.GptRequest;
+import org.enthusa.askdata.mapper.BiDataSourceMapper;
+import org.enthusa.askdata.mapper.BiFieldMapper;
+import org.enthusa.askdata.mapper.BiTableMapper;
+import org.enthusa.avatar.core.utils.RegexUtil;
+import org.enthusa.avatar.face.type.PageModel;
+import org.enthusa.avatar.face.type.Result;
+import org.enthusa.avatar.face.utils.ResultUtil;
+import org.enthusa.avatar.face.utils.Validate;
+import org.springframework.util.CollectionUtils;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.annotation.Resource;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * @author henry
+ * @date 2023/7/1
+ */
+@Slf4j
+@RestController
+@RequestMapping("/api/bi/venus")
+public class BiVenusController {
+    private static final Pattern LIMIT_PATTERN = Pattern.compile("limit\\s*\\d+[\\s\\d,;]*$");
+
+    @Resource
+    private BiDataSourceMapper biDataSourceMapper;
+
+    @Resource
+    private BiTableMapper biTableMapper;
+
+    @Resource
+    private BiFieldMapper biFieldMapper;
+
+    @Resource
+    private GptClient gptClient;
+
+    @PostMapping("/executeQuery")
+    public Result executeQuery(@RequestBody QueryVO queryVO) throws SQLException {
+        // Todo: 权限控制
+        if (StringUtils.containsAny(queryVO.getSql(), "pwd", "password", "email", "phone")) {
+            return ResultUtil.error(110, "包含敏感字段, 请修改 SQL 后再查询");
+        }
+
+        // Todo: 改写 SQL 还有很多细节, 比如分区检测
+        log.info("==> 改写前: {}", queryVO);
+        String sql = queryVO.getSql();
+        if (!RegexUtil.hasMatch(LIMIT_PATTERN, sql)) {
+            sql = sql.replaceFirst(";$", "") + " limit 1000";
+        }
+        log.info("==> 改写后: {}", sql);
+
+        // 数据源, 优先选数据表关联的数据源
+        Integer dsId = ObjectUtils.defaultIfNull(queryVO.getDs(), 1);
+        if (!CollectionUtils.isEmpty(queryVO.getTableIds())) {
+            Integer tableId = queryVO.getTableIds().get(0);
+            BiTable biTable = biTableMapper.selectByPrimaryKey(tableId);
+            dsId = biTable.getDsId();
+        }
+
+        QueryRunner qr = new QueryRunner();
+        Map<String, Object> model = new HashMap<>();
+        try (Connection conn = getConnection(dsId)) {
+            long begin = System.currentTimeMillis();
+            List<Map<String, Object>> data = qr.query(conn, sql, new MapListHandler());
+            long end = System.currentTimeMillis();
+            int elapsed = (int) (end - begin);
+            model.put("elapsed", elapsed);
+            if (CollectionUtils.isEmpty(data)) {
+                return ResultUtil.success(model);
+            }
+            model.put("propList", data.get(0).keySet());
+            model.put("data", data);
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+        return ResultUtil.success(model);
+    }
+
+    private Connection getConnection(Integer dsId) throws SQLException {
+        BiDataSource ds = biDataSourceMapper.selectByPrimaryKey(dsId);
+        byte[] bytes = Base64.getDecoder().decode(ds.getDetails());
+        Properties info = JSON.parseObject(new String(bytes), Properties.class);
+        return DriverManager.getConnection(info.getProperty("url"), info);
+
+    }
+
+    @PostMapping("/sqlTranslate")
+    public Result sqlTranslate(@RequestBody QueryVO queryVO) {
+        queryVO.setTableIds(Arrays.asList(1, 2, 3));
+        Validate.isEmpty("查询数据表", queryVO.getTableIds());
+        long begin = System.currentTimeMillis();
+        List<BiTable> tableList = biTableMapper.selectByIds(queryVO.getTableIds());
+        Integer dsId = tableList.get(0).getDsId();
+        List<String> tablesAndColumn = queryVO.getTablesAndColumn();
+        StringBuilder sb = new StringBuilder();
+        if (Objects.equals(dsId, 2)) {
+            sb.append("### Hive SQL tables, with their properties:\n");
+        } else {
+            sb.append("### MySQL tables, with their properties:\n");
+        }
+        sb.append("#\n");
+        for (BiTable table : tableList) {
+            sb.append(generateTableSchema(table, tablesAndColumn));
+        }
+        sb.append("#\n");
+        sb.append(String.format("### %s\n", queryVO.getQuestion()));
+        sb.append("SELECT\n");
+        String userMsg = sb.toString();
+        log.info("User msg: {}", userMsg);
+
+        GptRequest completion = GptRequest.newRequest("Translate natural language to SQL queries.");
+        completion.addUserMsg(userMsg);
+        String reply = gptClient.chatCompletion(completion);
+        String sql = String.format("SELECT %s", reply);
+        String res = sqlFormatFromDsAndSql(dsId, sql);
+        long end = System.currentTimeMillis();
+        Map<String, Object> model = new HashMap<>();
+        model.put("sql", res);
+        model.put("elapsed", end - begin);
+        return ResultUtil.success(model);
+    }
+
+    private String generateTableSchema(BiTable table, List<String> tablesAndColumn) {
+        String fieldNames;
+        if (!CollectionUtils.isEmpty(tablesAndColumn)) {
+            fieldNames = tablesAndColumn.stream().filter(t -> table.getId().equals(Integer.valueOf(t.split("\\.")[0]))).map(t -> t.split("\\.")[1]).collect(Collectors.joining(", "));
+        } else {
+            PageModel<BiField> pageModel = new PageModel<>();
+            pageModel.addCondition("table_id=?", table.getId());
+            pageModel.setOrders("column_seq");
+            List<BiField> fields = biFieldMapper.selectByPage(pageModel);
+            fieldNames = fields.stream().map(BiField::getName).collect(Collectors.joining(", "));
+        }
+        return String.format("# %s.%s(%s)\n", table.getCatalog(), table.getName(), fieldNames);
+    }
+
+    private String sqlFormatFromDsAndSql(Integer ds, String sql) {
+        if (ds != 1 && ds != 2) {
+            return sql;
+        }
+        return Objects.equals(2, ds) ? SQLUtils.formatOdps(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION) : SQLUtils.formatMySql(sql, SQLUtils.DEFAULT_LCASE_FORMAT_OPTION);
+    }
+
+    @Data
+    public static class QueryVO {
+        private List<Integer> tableIds;
+        private List<String> tablesAndColumn;
+        private String question;
+        private Integer ds;
+        private String sql;
+    }
+}

src/main/java/org/enthusa/askdata/dao/BiFieldDao.java

+package org.enthusa.askdata.dao;
+
+import org.enthusa.askdata.entity.BiField;
+
+/**
+ * @author henry
+ * @date 2022/3/19
+ */
+public interface BiFieldDao {
+    Integer save(BiField field);
+}

src/main/java/org/enthusa/askdata/dao/BiTableDao.java

+package org.enthusa.askdata.dao;
+
+import org.enthusa.askdata.entity.BiTable;
+
+/**
+ * @author henry
+ * @date 2022/3/18
+ */
+public interface BiTableDao {
+    Integer save(BiTable biTable);
+}

src/main/java/org/enthusa/askdata/dao/impl/BiFieldDaoImpl.java

+package org.enthusa.askdata.dao.impl;
+
+import org.enthusa.askdata.dao.BiFieldDao;
+import org.enthusa.askdata.entity.BiField;
+import org.enthusa.askdata.mapper.BiFieldMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+import java.util.Objects;
+
+/**
+ * @author henry
+ * @date 2022/3/19
+ */
+@Repository("biFieldDao")
+public class BiFieldDaoImpl implements BiFieldDao {
+    @Autowired(required = false)
+    private BiFieldMapper biFieldMapper;
+
+    @Override
+    public Integer save(BiField field) {
+        BiField biField = biFieldMapper.selectByTableIdAndName(field.getTableId(), field.getName());
+        if (Objects.isNull(biField)) {
+            biField = new BiField();
+            biField.setTableId(field.getTableId());
+            biField.setName(field.getName());
+        }
+        biField.setStatus(field.getStatus());
+        biField.setColumnSeq(field.getColumnSeq());
+        biField.setPrimaryKeySeq(field.getPrimaryKeySeq());
+        biField.setReferId(field.getReferId());
+        biField.setBrief(field.getBrief());
+        if (Objects.isNull(biField.getUpdateUserId()) || Objects.nonNull(field.getUpdateUserId())) {
+            biField.setRemarks(field.getRemarks());
+        }
+        biField.setTypeName(field.getTypeName());
+        biField.setColumnSize(field.getColumnSize());
+        biField.setIsAutoIncrement(field.getIsAutoIncrement());
+        biField.setIsNullable(field.getIsNullable());
+        biField.setDefaultValue(field.getDefaultValue());
+        biField.setIsEnumerable(field.getIsEnumerable());
+        biField.setOptions(field.getOptions());
+        biField.setUpdateUserId(field.getUpdateUserId());
+        if (Objects.isNull(biField.getId())) {
+            biFieldMapper.insertSelective(biField);
+        } else {
+            biFieldMapper.updateByPrimaryKeySelective(biField);
+        }
+        return biField.getId();
+    }
+}

src/main/java/org/enthusa/askdata/dao/impl/BiTableDaoImpl.java

+package org.enthusa.askdata.dao.impl;
+
+import org.enthusa.askdata.dao.BiTableDao;
+import org.enthusa.askdata.entity.BiTable;
+import org.enthusa.askdata.mapper.BiTableMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+import java.util.Objects;
+
+/**
+ * @author henry
+ * @date 2022/3/18
+ */
+@Repository("biTableDao")
+public class BiTableDaoImpl implements BiTableDao {
+    @Autowired(required = false)
+    private BiTableMapper biTableMapper;
+
+    @Override
+    public Integer save(BiTable table) {
+        BiTable biTable = biTableMapper.selectByDsIdAndCatalogAndName(table.getDsId(), table.getCatalog(), table.getName());
+        if (Objects.isNull(biTable)) {
+            biTable = new BiTable();
+            biTable.setDsId(table.getDsId());
+            biTable.setCatalog(table.getCatalog());
+            biTable.setName(table.getName());
+        }
+        biTable.setStatus(table.getStatus());
+        biTable.setBrief(table.getBrief());
+        if (Objects.isNull(biTable.getUpdateUserId()) || Objects.nonNull(table.getUpdateUserId())) {
+            biTable.setRemarks(table.getRemarks());
+        }
+        biTable.setOwnerId(table.getOwnerId());
+        biTable.setUpdateUserId(table.getUpdateUserId());
+        if (Objects.isNull(biTable.getId())) {
+            biTableMapper.insertSelective(biTable);
+        } else {
+            biTableMapper.updateByPrimaryKeySelective(biTable);
+        }
+        return biTable.getId();
+    }
+}

src/main/java/org/enthusa/askdata/entity/BiDataSource.java

+package org.enthusa.askdata.entity;
+
+import lombok.Data;
+import org.enthusa.avatar.mybatis.annotation.Transient;
+
+import java.util.Date;
+import java.util.List;
+
+@Data
+public class BiDataSource {
+    /**
+     * id
+     */
+    private Integer id;
+
+    /**
+     * 数据源名称
+     */
+    private String name;
+
+    /**
+     * 数据源连接信息
+     */
+    private String details;
+
+    @Transient
+    private String url;
+    @Transient
+    private String user;
+    @Transient
+    private String password;
+
+    /**
+     * 该数据源下, 可用的数据库, 多个逗号连接
+     */
+    private String catalogs;
+
+    @Transient
+    private List<String> catalogList;
+
+    /**
+     * create_time
+     */
+    private Date createTime;
+
+    /**
+     * update_time
+     */
+    private Date updateTime;
+
+    public void setName(String name) {
+        this.name = name == null ? null : name.trim();
+    }
+
+    public void setDetails(String details) {
+        this.details = details == null ? null : details.trim();
+    }
+
+    public void setCatalogs(String catalogs) {
+        this.catalogs = catalogs == null ? null : catalogs.trim();
+    }
+}

src/main/java/org/enthusa/askdata/entity/BiField.java

+package org.enthusa.askdata.entity;
+
+import lombok.Data;
+
+import java.util.Date;
+
+@Data
+public class BiField {
+    /**
+     * id
+     */
+    private Integer id;
+
+    /**
+     * 数据表 id
+     */
+    private Integer tableId;
+
+    /**
+     * 列顺序, 序号
+     */
+    private Integer columnSeq;
+
+    /**
+     * 主键顺序: 0, 表示非主键
+     */
+    private Integer primaryKeySeq;
+
+    /**
+     * 外键, 参考字段
+     */
+    private Integer referId;
+
+    /**
+     * 字段名, 英文名
+     */
+    private String name;
+
+    /**
+     * 字段简介, 中文名
+     */
+    private String brief;
+
+    /**
+     * 字段注释, 详解
+     */
+    private String remarks;
+
+    /**
+     * 字段类型
+     */
+    private String typeName;
+
+    /**
+     * 字段长度
+     */
+    private Integer columnSize;
+
+    /**
+     * 是否自增
+     */
+    private Boolean isAutoIncrement;
+
+    /**
+     * 是否允许为空
+     */
+    private Boolean isNullable;
+
+    /**
+     * 字段默认值
+     */
+    private String defaultValue;
+
+    /**
+     * 是否可枚举
+     */
+    private Boolean isEnumerable;
+
+    /**
+     * 枚举值列表, 格式: value->label
+     */
+    private String options;
+
+    /**
+     * 最近修改人
+     */
+    private Integer updateUserId;
+
+    /**
+     * create_time
+     */
+    private Date createTime;
+
+    /**
+     * update_time
+     */
+    private Date updateTime;
+
+    /**
+     * 是否生效： 0:生效； 1:失效
+     */
+    private Integer status;
+
+    public void setName(String name) {
+        this.name = name == null ? null : name.trim();
+    }
+
+    public void setBrief(String brief) {
+        this.brief = brief == null ? null : brief.trim();
+    }
+
+    public void setRemarks(String remarks) {
+        this.remarks = remarks == null ? null : remarks.trim();
+    }
+
+    public void setTypeName(String typeName) {
+        this.typeName = typeName == null ? null : typeName.trim();
+    }
+
+    public void setDefaultValue(String defaultValue) {
+        this.defaultValue = defaultValue == null ? null : defaultValue.trim();
+    }
+
+    public void setOptions(String options) {
+        this.options = options == null ? null : options.trim();
+    }
+}

src/main/java/org/enthusa/askdata/entity/BiTable.java

+package org.enthusa.askdata.entity;
+
+import lombok.Data;
+import org.enthusa.avatar.mybatis.annotation.Transient;
+
+import java.util.Date;
+
+@Data
+public class BiTable {
+    /**
+     * id
+     */
+    private Integer id;
+
+    /**
+     * 数据源 id
+     */
+    private Integer dsId;
+
+    /**
+     * 目录, 相当于数据库
+     */
+    private String catalog;
+
+    /**
+     * 表名, 英文名
+     */
+    private String name;
+
+    /**
+     * 表简介, 中文名
+     */
+    private String brief;
+
+    /**
+     * 表注释, 详解
+     */
+    private String remarks;
+
+    /**
+     * 责任人
+     */
+    private Integer ownerId;
+
+    /**
+     * 最近修改人
+     */
+    private Integer updateUserId;
+
+    /**
+     * create_time
+     */
+    private Date createTime;
+
+    /**
+     * update_time
+     */
+    private Date updateTime;
+
+    /**
+     * 是否生效： 0:生效； 1:失效
+     */
+    private Integer status;
+
+
+    /**
+     * query_cnt 近三个月查询次数
+     */
+    private Integer queryCnt;
+
+    /**
+     * query_user_cnt 近三个月查询人数
+     */
+    private Integer queryUserCnt;
+
+    /**
+     * last_query_time
+     */
+    private Date lastQueryTime;
+
+    @Transient
+    private String columnNames = "";
+    @Transient
+    private String columnBriefs = "";
+    @Transient
+    private String columnRemarks = "";
+
+    public void setCatalog(String catalog) {
+        this.catalog = catalog == null ? null : catalog.trim();
+    }
+
+    public void setName(String name) {
+        this.name = name == null ? null : name.trim();
+    }
+
+    public void setBrief(String brief) {
+        this.brief = brief == null ? null : brief.trim();
+    }
+
+    public void setRemarks(String remarks) {
+        this.remarks = remarks == null ? null : remarks.trim();
+    }
+}

src/main/java/org/enthusa/askdata/ext/inscode/GptClient.java

+package org.enthusa.askdata.ext.inscode;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import okhttp3.*;
+import org.apache.http.HttpHeaders;
+import org.enthusa.avatar.core.consts.TextConstant;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * @author henry
+ * @date 2023/7/1
+ */
+@Slf4j
+public class GptClient {
+    private OkHttpClient httpClient;
+
+    private GptClient(Builder builder) {
+        httpClient = new OkHttpClient.Builder()
+                .connectTimeout(10, TimeUnit.SECONDS)
+                .writeTimeout(50, TimeUnit.SECONDS)
+                .readTimeout(50, TimeUnit.SECONDS)
+                .build();
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static final class Builder {
+        public GptClient build() {
+            return new GptClient(this);
+        }
+    }
+
+    public String chatCompletion(GptRequest completion) {
+        String json = JSON.toJSONString(completion);
+
+        RequestBody requestBody = RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json);
+        Request request = new Request.Builder()
+                .url("https://inscode-api.csdn.net/api/v1/gpt/")
+                .post(requestBody)
+                .header(HttpHeaders.CONTENT_TYPE, "application/json")
+                .header(HttpHeaders.ACCEPT_CHARSET, "utf-8")
+                .header(HttpHeaders.CONNECTION, "Keep-Alive")
+                .build();
+        try {
+            Response response = httpClient.newCall(request).execute();
+            if (response.isSuccessful()) {
+                String text = response.body().string();
+                return TextConstant.LINE_SPLITTER.splitToStream(text).map(data -> data.substring(5)).filter(data -> !"[DONE]".equals(data)).map(data -> {
+                    JSONObject obj = JSON.parseObject(data);
+                    JSONObject delta = obj.getJSONArray("choices").getJSONObject(0).getJSONObject("delta");
+                    return delta.getString("content");
+                }).filter(Objects::nonNull).collect(Collectors.joining());
+            }
+        } catch (IOException e) {
+            log.error(e.getMessage(), e);
+        }
+        return "";
+    }
+}

src/main/java/org/enthusa/askdata/ext/inscode/GptRequest.java

+package org.enthusa.askdata.ext.inscode;
+
+import com.alibaba.fastjson.annotation.JSONField;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author henry
+ * @date 2023/7/1
+ */
+@Data
+public class GptRequest {
+    private enum Role {
+        System, Assistant, User
+    }
+
+    private List<Message> messages;
+    @JsonProperty("apikey")
+    @JSONField(name = "apikey")
+    private String apiKey;
+
+    private GptRequest() {
+        messages = new ArrayList<>();
+    }
+
+    private void addQueryWithRole(Role role, String query) {
+        Message msg = new Message();
+        msg.setRole(role.name().toLowerCase());
+        msg.setContent(query);
+        messages.add(msg);
+    }
+
+    public void addUserMsg(String msg) {
+        addQueryWithRole(Role.User, msg);
+    }
+
+    public void addAssistantMsg(String msg) {
+        addQueryWithRole(Role.Assistant, msg);
+    }
+
+    public static GptRequest newRequest(String systemDesc) {
+        GptRequest request = new GptRequest();
+        request.addQueryWithRole(Role.System, systemDesc);
+        return request;
+    }
+
+    @Data
+    public static class Message {
+        private String role;
+        private String content;
+    }
+}

src/main/java/org/enthusa/askdata/mapper/BiDataSourceMapper.java

+package org.enthusa.askdata.mapper;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.enthusa.askdata.entity.BiDataSource;
+import org.enthusa.avatar.mybatis.mapper.SqlMapper;
+
+@Mapper
+public interface BiDataSourceMapper extends SqlMapper<BiDataSource, Integer> {
+    int insertSelective(BiDataSource record);
+
+    int updateByPrimaryKeySelective(BiDataSource record);
+}

src/main/java/org/enthusa/askdata/mapper/BiFieldMapper.java

+package org.enthusa.askdata.mapper;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import org.enthusa.askdata.entity.BiField;
+import org.enthusa.avatar.mybatis.mapper.SqlMapper;
+
+import java.util.List;
+
+@Mapper
+public interface BiFieldMapper extends SqlMapper<BiField, Integer> {
+    List<BiField> selectByNameWithInfo(@Param("name") String name);
+
+    List<Integer> selectTableIdsByName(@Param("name") String name);
+
+    BiField selectByTableIdAndName(@Param("tableId") Integer tableId, @Param("name") String name);
+
+    int insertSelective(BiField record);
+
+    int updateByPrimaryKeySelective(BiField record);
+
+    int batchUpdateStatusByPrimaryKeys(@Param("list") List<Integer> ids);
+
+    List<BiField> selectByTableIds(@Param("list") List tableIds);
+
+}

src/main/java/org/enthusa/askdata/mapper/BiPostMapper.java

 package org.enthusa.askdata.mapper;
 
-import org.enthusa.askdata.entity.BiPost;
 import org.apache.ibatis.annotations.Mapper;
+import org.enthusa.askdata.entity.BiPost;
 import org.enthusa.avatar.mybatis.mapper.SqlMapper;
 
 @Mapper

src/main/java/org/enthusa/askdata/mapper/BiTableMapper.java

+package org.enthusa.askdata.mapper;
+
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+import org.enthusa.askdata.entity.BiTable;
+import org.enthusa.avatar.mybatis.mapper.SqlMapper;
+
+import java.util.List;
+
+@Mapper
+public interface BiTableMapper extends SqlMapper<BiTable, Integer> {
+    BiTable selectByDsIdAndCatalogAndName(@Param("dsId") Integer dsId, @Param("catalog") String catalog, @Param("name") String name);
+
+    int insertSelective(BiTable record);
+
+    int updateByPrimaryKeySelective(BiTable record);
+
+    int batchUpdateStatusByPrimaryKeys(@Param("list") List<Integer> ids);
+
+    int batchUpdateHotValue(@Param("list") List<BiTable> tables);
+}

src/main/java/org/enthusa/askdata/task/AbstractTask.java

+package org.enthusa.askdata.task;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.enthusa.avatar.utils.task.Task;
+import org.enthusa.avatar.utils.task.TaskModel;
+
+import java.util.Date;
+
+/**
+ * @author henry
+ * @date 2021/9/8
+ */
+@Slf4j
+public abstract class AbstractTask implements Task {
+    @Setter
+    @Getter
+    private String[] args = new String[]{};
+
+    @Override
+    public String getBiz() {
+        return "askdata-task";
+    }
+
+    @Override
+    public String getTaskName() {
+        return getClass().getSimpleName();
+    }
+
+    @Override
+    public void start() {
+        TaskModel model = new TaskModel(getBiz(), getTaskName());
+        log.info("Begin to run {} at {}", getTaskName(), new Date());
+        run(model);
+        log.info("Complete {} at {}", getTaskName(), new Date());
+    }
+}

src/main/java/org/enthusa/askdata/task/impl/FillMetaDataTask.java

+package org.enthusa.askdata.task.impl;
+
+import com.alibaba.fastjson.JSON;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.enthusa.askdata.dao.BiFieldDao;
+import org.enthusa.askdata.dao.BiTableDao;
+import org.enthusa.askdata.entity.BiDataSource;
+import org.enthusa.askdata.entity.BiField;
+import org.enthusa.askdata.entity.BiTable;
+import org.enthusa.askdata.mapper.BiDataSourceMapper;
+import org.enthusa.askdata.mapper.BiFieldMapper;
+import org.enthusa.askdata.mapper.BiTableMapper;
+import org.enthusa.askdata.task.AbstractTask;
+import org.enthusa.avatar.core.consts.TextConstant;
+import org.enthusa.avatar.db.metadata.ColumnEntity;
+import org.enthusa.avatar.db.metadata.MetaDataUtils;
+import org.enthusa.avatar.db.metadata.TableEntity;
+import org.enthusa.avatar.utils.task.TaskModel;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import javax.annotation.Resource;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.DriverManager;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @author henry
+ * @date 2022/3/21
+ */
+@Slf4j
+@Component
+public class FillMetaDataTask extends AbstractTask {
+    @Resource
+    private BiDataSourceMapper biDataSourceMapper;
+
+    @Resource
+    private BiTableDao biTableDao;
+
+    @Resource
+    private BiFieldDao biFieldDao;
+
+    @Resource
+    private BiTableMapper biTableMapper;
+
+    @Resource
+    private BiFieldMapper biFieldMapper;
+
+    @Override
+    public void run(TaskModel taskModel) {
+        List<Integer> biTablesBefore = biTableMapper.selectAll().stream().filter(t -> t.getStatus() == 0).map(BiTable::getId).collect(Collectors.toList());
+        Set<Integer> biTablesAfter = new HashSet<>();
+        List<Integer> biFieldsBefore = biFieldMapper.selectAll().stream().filter(t -> t.getStatus() == 0).map(BiField::getId).collect(Collectors.toList());
+        Set<Integer> biFieldsAfter = new HashSet<>();
+        List<BiDataSource> dsList = biDataSourceMapper.selectAll();
+        for (BiDataSource ds : dsList) {
+            byte[] bytes = Base64.getDecoder().decode(ds.getDetails());
+            Properties info = JSON.parseObject(new String(bytes), Properties.class);
+            List<String> catalogs = TextConstant.COMMA_SPLITTER.splitToList(ds.getCatalogs());
+            try (Connection conn = DriverManager.getConnection(info.getProperty("url"), info)) {
+                DatabaseMetaData metaData = conn.getMetaData();
+                for (String catalog : catalogs) {
+                    List<TableEntity> tables = MetaDataUtils.getTables(metaData, catalog);
+                    for (TableEntity table : tables) {
+                        if (StringUtils.containsAny(table.getName(), "tmp", "temp", "bak", "backup")
+                                && !StringUtils.startsWithAny(table.getName(), "ods", "dwd", "dwm", "dws", "dim", "app")) {
+                            log.warn("{}, skipped!", table);
+                            continue;
+                        }
+                        log.info("{}", table);
+                        BiTable biTable = new BiTable();
+                        biTable.setDsId(ds.getId());
+                        biTable.setCatalog(table.getCatalog());
+                        biTable.setName(table.getName());
+                        biTable.setRemarks(table.getRemarks());
+                        biTable.setStatus(0);
+                        Integer tableId = biTableDao.save(biTable);
+                        biTablesAfter.add(tableId);
+                        List<ColumnEntity> columns = MetaDataUtils.getColumns(metaData, catalog, table.getName());
+                        for (ColumnEntity column : columns) {
+                            BiField biField = new BiField();
+                            biField.setTableId(tableId);
+                            biField.setColumnSeq(column.getColumnSeq());
+                            biField.setPrimaryKeySeq(column.getPrimaryKeySeq());
+                            biField.setName(column.getName());
+                            biField.setRemarks(column.getRemarks());
+                            biField.setTypeName(column.getTypeName());
+                            biField.setColumnSize(column.getColumnSize());
+                            biField.setIsAutoIncrement(column.getIsAutoIncrement());
+                            biField.setIsNullable(column.getIsNullable());
+                            biField.setDefaultValue(column.getDefaultValue());
+                            biField.setStatus(0);
+                            Integer fieldId = biFieldDao.save(biField);
+                            biFieldsAfter.add(fieldId);
+                        }
+                    }
+                }
+            } catch (Exception e) {
+                log.error(e.getMessage(), e);
+            }
+        }
+        List<Integer> biTables = biTablesBefore.stream().filter(t -> !biTablesAfter.contains(t)).collect(Collectors.toList());
+        if (!CollectionUtils.isEmpty(biTables)) {
+            biTableMapper.batchUpdateStatusByPrimaryKeys(biTables);
+        }
+        List<Integer> biFields = biFieldsBefore.stream().filter(t -> !biFieldsAfter.contains(t)).collect(Collectors.toList());
+        if (!CollectionUtils.isEmpty(biFields)) {
+            biFieldMapper.batchUpdateStatusByPrimaryKeys(biFields);
+        }
+    }
+}

src/main/resources/application-dev.yml

 settings:
   env: dev
+  openai:
+    host: from_command_line
+    key: from_command_line
 spring:
   datasource:
     driver-class-name: com.mysql.cj.jdbc.Driver

src/main/resources/application-prod.yml

 settings:
   env: prod
+  openai:
+    host: from_command_line
+    key: from_command_line
 spring:
   datasource:
     driver-class-name: com.mysql.cj.jdbc.Driver

src/main/resources/import.sql

-CREATE TABLE bi_post (
-  id          INT               AUTO_INCREMENT PRIMARY KEY,
-  title       VARCHAR(255),
-  create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+DROP TABLE IF EXISTS `bi_post`;
+CREATE TABLE `bi_post` (
+  `id`          INT      NOT NULL AUTO_INCREMENT,
+  `title`       VARCHAR(255)      DEFAULT NULL,
+  `create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`)
 );
+INSERT INTO bi_post (title) VALUE ('测试帖子');
+
+DROP TABLE IF EXISTS `bi_data_source`;
+CREATE TABLE `bi_data_source` (
+  `id`          INT(11)     NOT NULL AUTO_INCREMENT,
+  `name`        VARCHAR(64) NOT NULL COMMENT '数据源名称',
+  `details`     TEXT COMMENT '数据源连接信息',
+  `catalogs`    VARCHAR(255)         DEFAULT NULL COMMENT '该数据源下, 可用的数据库, 多个逗号连接',
+  `create_time` DATETIME    NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `update_time` DATETIME    NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `idx_update_time` (`update_time`)
+);
+
+DROP TABLE IF EXISTS `bi_table`;
+CREATE TABLE `bi_table` (
+  `id`              INT(11)      NOT NULL AUTO_INCREMENT,
+  `ds_id`           INT(11)      NOT NULL COMMENT '数据源 id',
+  `catalog`         VARCHAR(32)  NOT NULL COMMENT '目录, 相当于数据库',
+  `name`            VARCHAR(128) NOT NULL COMMENT '表名, 英文名',
+  `brief`           VARCHAR(16)           DEFAULT NULL COMMENT '表简介, 中文名',
+  `remarks`         TEXT COMMENT '表注释, 详解',
+  `owner_id`        INT(11)               DEFAULT NULL COMMENT '责任人',
+  `update_user_id`  INT(11)               DEFAULT NULL COMMENT '最近修改人',
+  `create_time`     TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `update_time`     TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `status`          INT(11)               DEFAULT '0' COMMENT '是否生效 0生效 1失效',
+  `query_cnt`       INT(11)      NOT NULL DEFAULT '0' COMMENT '近三个月查询次数',
+  `query_user_cnt`  INT(11)      NOT NULL DEFAULT '0' COMMENT '近三个月查询人数',
+  `last_query_time` DATETIME              DEFAULT NULL COMMENT '最后一次查询时间',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `udx_on_ds_catalog_name` (`ds_id`, `catalog`, `name`),
+  KEY `idx_update_time` (`update_time`)
+);
+
+DROP TABLE IF EXISTS `bi_field`;
+CREATE TABLE `bi_field` (
+  `id`                INT(11)     NOT NULL AUTO_INCREMENT,
+  `table_id`          INT(11)     NOT NULL COMMENT '数据表 id',
+  `column_seq`        INT(11)     NOT NULL COMMENT '列顺序, 序号',
+  `primary_key_seq`   INT(11)     NOT NULL DEFAULT '0' COMMENT '主键顺序: 0, 表示非主键',
+  `refer_id`          INT(11)              DEFAULT NULL COMMENT '外键, 参考字段',
+  `name`              VARCHAR(64) NOT NULL COMMENT '字段名, 英文名',
+  `brief`             VARCHAR(16)          DEFAULT NULL COMMENT '字段简介, 中文名',
+  `remarks`           TEXT COMMENT '字段注释, 详解',
+  `type_name`         VARCHAR(32)          DEFAULT NULL COMMENT '字段类型',
+  `column_size`       INT(11)              DEFAULT NULL COMMENT '字段长度',
+  `is_auto_increment` TINYINT(1)           DEFAULT NULL COMMENT '是否自增',
+  `is_nullable`       TINYINT(1)           DEFAULT NULL COMMENT '是否允许为空',
+  `default_value`     VARCHAR(64)          DEFAULT NULL COMMENT '字段默认值',
+  `is_enumerable`     TINYINT(1)           DEFAULT NULL COMMENT '是否可枚举',
+  `options`           TEXT COMMENT '枚举值列表, 格式: value->label',
+  `update_user_id`    INT(11)              DEFAULT NULL COMMENT '最近修改人',
+  `create_time`       TIMESTAMP   NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `update_time`       TIMESTAMP   NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  `status`            INT(11)              DEFAULT '0' COMMENT '是否生效 0生效 1失效',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `udx_on_table_name` (`table_id`, `name`),
+  KEY `idx_update_time` (`update_time`),
+  KEY `idx_on_name` (`name`)
+);
\ No newline at end of file

src/main/resources/org/enthusa/askdata/mapper/BiDataSourceMapper.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.enthusa.askdata.mapper.BiDataSourceMapper">
+  <resultMap id="BaseResultMap" type="org.enthusa.askdata.entity.BiDataSource">
+    <id column="id" property="id" jdbcType="INTEGER"/>
+    <result column="name" property="name" jdbcType="VARCHAR"/>
+    <result column="details" property="details" jdbcType="VARCHAR"/>
+    <result column="catalogs" property="catalogs" jdbcType="VARCHAR"/>
+    <result column="create_time" property="createTime" jdbcType="TIMESTAMP"/>
+    <result column="update_time" property="updateTime" jdbcType="TIMESTAMP"/>
+  </resultMap>
+  <sql id="Base_Column_List">
+    id, name, details, catalogs, create_time, update_time
+  </sql>
+  <insert id="insertSelective" parameterType="org.enthusa.askdata.entity.BiDataSource"
+          useGeneratedKeys="true" keyProperty="id">
+    insert into bi_data_source
+    <trim prefix="(" suffix=")" suffixOverrides=",">
+      <if test="name != null">
+        name,
+      </if>
+      <if test="details != null">
+        details,
+      </if>
+      <if test="catalogs != null">
+        catalogs,
+      </if>
+    </trim>
+    <trim prefix="values (" suffix=")" suffixOverrides=",">
+      <if test="name != null">
+        #{name,jdbcType=VARCHAR},
+      </if>
+      <if test="details != null">
+        #{details,jdbcType=VARCHAR},
+      </if>
+      <if test="catalogs != null">
+        #{catalogs,jdbcType=VARCHAR},
+      </if>
+    </trim>
+  </insert>
+  <update id="updateByPrimaryKeySelective" parameterType="org.enthusa.askdata.entity.BiDataSource">
+    update bi_data_source
+    <set>
+      <if test="name != null">
+        name = #{name,jdbcType=VARCHAR},
+      </if>
+      <if test="details != null">
+        details = #{details,jdbcType=VARCHAR},
+      </if>
+      <if test="catalogs != null">
+        catalogs = #{catalogs,jdbcType=VARCHAR},
+      </if>
+    </set>
+    where id = #{id,jdbcType=INTEGER}
+  </update>
+</mapper>

src/main/resources/org/enthusa/askdata/mapper/BiFieldMapper.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.enthusa.askdata.mapper.BiFieldMapper">
+  <resultMap id="BaseResultMap" type="org.enthusa.askdata.entity.BiField">
+    <id column="id" property="id" jdbcType="INTEGER"/>
+    <result column="table_id" property="tableId" jdbcType="INTEGER"/>
+    <result column="column_seq" property="columnSeq" jdbcType="INTEGER"/>
+    <result column="primary_key_seq" property="primaryKeySeq" jdbcType="INTEGER"/>
+    <result column="refer_id" property="referId" jdbcType="INTEGER"/>
+    <result column="name" property="name" jdbcType="VARCHAR"/>
+    <result column="brief" property="brief" jdbcType="VARCHAR"/>
+    <result column="remarks" property="remarks" jdbcType="VARCHAR"/>
+    <result column="type_name" property="typeName" jdbcType="VARCHAR"/>
+    <result column="column_size" property="columnSize" jdbcType="INTEGER"/>
+    <result column="is_auto_increment" property="isAutoIncrement" jdbcType="BIT"/>
+    <result column="is_nullable" property="isNullable" jdbcType="BIT"/>
+    <result column="default_value" property="defaultValue" jdbcType="VARCHAR"/>
+    <result column="is_enumerable" property="isEnumerable" jdbcType="BIT"/>
+    <result column="options" property="options" jdbcType="VARCHAR"/>
+    <result column="update_user_id" property="updateUserId" jdbcType="INTEGER"/>
+    <result column="create_time" property="createTime" jdbcType="TIMESTAMP"/>
+    <result column="update_time" property="updateTime" jdbcType="TIMESTAMP"/>
+    <result column="status" property="status" jdbcType="INTEGER"/>
+  </resultMap>
+  <sql id="Base_Column_List">
+    id, table_id, column_seq, primary_key_seq, refer_id, name, brief, remarks, type_name, column_size, is_auto_increment, is_nullable, default_value, is_enumerable, options, update_user_id, create_time, update_time, status
+  </sql>
+  <select id="selectByNameWithInfo" resultMap="BaseResultMap">
+    select
+    <include refid="Base_Column_List"/>
+    from bi_field
+    where name = #{name,jdbcType=VARCHAR}
+    and (length(brief) > 0 or length(remarks) > 0)
+  </select>
+  <select id="selectByTableIdAndName" resultMap="BaseResultMap">
+    select
+    <include refid="Base_Column_List"/>
+    from bi_field
+    where table_id = #{tableId,jdbcType=INTEGER}
+    and name = #{name,jdbcType=VARCHAR}
+    limit 1
+  </select>
+  <select id="selectTableIdsByName" resultType="java.lang.Integer">
+    select
+    distinct table_id
+    from bi_field
+    where name = #{name,jdbcType=VARCHAR}
+  </select>
+  <select id="selectByTableIds" resultMap="BaseResultMap">
+    select * from bi_field
+    where table_id in
+    <foreach item='item' collection='list' open='(' separator=',' close=')'>
+      #{item,jdbcType=BIGINT}
+    </foreach>
+  </select>
+  <insert id="insertSelective" parameterType="org.enthusa.askdata.entity.BiField"
+          useGeneratedKeys="true" keyProperty="id">
+    insert into bi_field
+    <trim prefix="(" suffix=")" suffixOverrides=",">
+      <if test="tableId != null">
+        table_id,
+      </if>
+      <if test="columnSeq != null">
+        column_seq,
+      </if>
+      <if test="primaryKeySeq != null">
+        primary_key_seq,
+      </if>
+      <if test="referId != null">
+        refer_id,
+      </if>
+      <if test="name != null">
+        name,
+      </if>
+      <if test="brief != null">
+        brief,
+      </if>
+      <if test="remarks != null">
+        remarks,
+      </if>
+      <if test="typeName != null">
+        type_name,
+      </if>
+      <if test="columnSize != null">
+        column_size,
+      </if>
+      <if test="isAutoIncrement != null">
+        is_auto_increment,
+      </if>
+      <if test="isNullable != null">
+        is_nullable,
+      </if>
+      <if test="defaultValue != null">
+        default_value,
+      </if>
+      <if test="isEnumerable != null">
+        is_enumerable,
+      </if>
+      <if test="options != null">
+        options,
+      </if>
+      <if test="updateUserId != null">
+        update_user_id,
+      </if>
+      <if test="status != null">
+        status,
+      </if>
+    </trim>
+    <trim prefix="values (" suffix=")" suffixOverrides=",">
+      <if test="tableId != null">
+        #{tableId,jdbcType=INTEGER},
+      </if>
+      <if test="columnSeq != null">
+        #{columnSeq,jdbcType=INTEGER},
+      </if>
+      <if test="primaryKeySeq != null">
+        #{primaryKeySeq,jdbcType=INTEGER},
+      </if>
+      <if test="referId != null">
+        #{referId,jdbcType=INTEGER},
+      </if>
+      <if test="name != null">
+        #{name,jdbcType=VARCHAR},
+      </if>
+      <if test="brief != null">
+        #{brief,jdbcType=VARCHAR},
+      </if>
+      <if test="remarks != null">
+        #{remarks,jdbcType=VARCHAR},
+      </if>
+      <if test="typeName != null">
+        #{typeName,jdbcType=VARCHAR},
+      </if>
+      <if test="columnSize != null">
+        #{columnSize,jdbcType=INTEGER},
+      </if>
+      <if test="isAutoIncrement != null">
+        #{isAutoIncrement,jdbcType=BIT},
+      </if>
+      <if test="isNullable != null">
+        #{isNullable,jdbcType=BIT},
+      </if>
+      <if test="defaultValue != null">
+        #{defaultValue,jdbcType=VARCHAR},
+      </if>
+      <if test="isEnumerable != null">
+        #{isEnumerable,jdbcType=BIT},
+      </if>
+      <if test="options != null">
+        #{options,jdbcType=VARCHAR},
+      </if>
+      <if test="updateUserId != null">
+        #{updateUserId,jdbcType=INTEGER},
+      </if>
+      <if test="status != null">
+        #{status,jdbcType=INTEGER},
+      </if>
+    </trim>
+  </insert>
+  <update id="updateByPrimaryKeySelective" parameterType="org.enthusa.askdata.entity.BiField">
+    update bi_field
+    <set>
+      <if test="tableId != null">
+        table_id = #{tableId,jdbcType=INTEGER},
+      </if>
+      <if test="columnSeq != null">
+        column_seq = #{columnSeq,jdbcType=INTEGER},
+      </if>
+      <if test="primaryKeySeq != null">
+        primary_key_seq = #{primaryKeySeq,jdbcType=INTEGER},
+      </if>
+      <if test="referId != null">
+        refer_id = #{referId,jdbcType=INTEGER},
+      </if>
+      <if test="name != null">
+        name = #{name,jdbcType=VARCHAR},
+      </if>
+      <if test="brief != null">
+        brief = #{brief,jdbcType=VARCHAR},
+      </if>
+      <if test="remarks != null">
+        remarks = #{remarks,jdbcType=VARCHAR},
+      </if>
+      <if test="typeName != null">
+        type_name = #{typeName,jdbcType=VARCHAR},
+      </if>
+      <if test="columnSize != null">
+        column_size = #{columnSize,jdbcType=INTEGER},
+      </if>
+      <if test="isAutoIncrement != null">
+        is_auto_increment = #{isAutoIncrement,jdbcType=BIT},
+      </if>
+      <if test="isNullable != null">
+        is_nullable = #{isNullable,jdbcType=BIT},
+      </if>
+      <if test="defaultValue != null">
+        default_value = #{defaultValue,jdbcType=VARCHAR},
+      </if>
+      <if test="isEnumerable != null">
+        is_enumerable = #{isEnumerable,jdbcType=BIT},
+      </if>
+      <if test="options != null">
+        options = #{options,jdbcType=VARCHAR},
+      </if>
+      <if test="updateUserId != null">
+        update_user_id = #{updateUserId,jdbcType=INTEGER},
+      </if>
+      <if test="status != null">
+        status = #{status,jdbcType=INTEGER},
+      </if>
+    </set>
+    where id = #{id,jdbcType=INTEGER}
+  </update>
+  <update id="batchUpdateStatusByPrimaryKeys">
+    update bi_field set status = 1
+    where id in
+    <foreach item='item' collection='list' open='(' separator=',' close=')'>
+      #{item,jdbcType=BIGINT}
+    </foreach>
+  </update>
+</mapper>

src/main/resources/org/enthusa/askdata/mapper/BiTableMapper.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="org.enthusa.askdata.mapper.BiTableMapper">
+  <resultMap id="BaseResultMap" type="org.enthusa.askdata.entity.BiTable">
+    <id column="id" property="id" jdbcType="INTEGER"/>
+    <result column="ds_id" property="dsId" jdbcType="INTEGER"/>
+    <result column="catalog" property="catalog" jdbcType="VARCHAR"/>
+    <result column="name" property="name" jdbcType="VARCHAR"/>
+    <result column="brief" property="brief" jdbcType="VARCHAR"/>
+    <result column="remarks" property="remarks" jdbcType="VARCHAR"/>
+    <result column="owner_id" property="ownerId" jdbcType="INTEGER"/>
+    <result column="update_user_id" property="updateUserId" jdbcType="INTEGER"/>
+    <result column="create_time" property="createTime" jdbcType="TIMESTAMP"/>
+    <result column="update_time" property="updateTime" jdbcType="TIMESTAMP"/>
+    <result column="status" property="status" jdbcType="INTEGER"/>
+    <result column="query_cnt" property="queryCnt" jdbcType="INTEGER"/>
+    <result column="query_user_cnt" property="queryUserCnt" jdbcType="INTEGER"/>
+    <result column="last_query_time" property="lastQueryTime" jdbcType="TIMESTAMP"/>
+  </resultMap>
+  <sql id="Base_Column_List">
+    id, ds_id, catalog, name, brief, remarks, owner_id, update_user_id, create_time, update_time, status, query_cnt,
+    query_user_cnt, last_query_time
+  </sql>
+  <select id="selectByDsIdAndCatalogAndName" resultMap="BaseResultMap">
+    select
+    <include refid="Base_Column_List"/>
+    from bi_table
+    where ds_id = #{dsId,jdbcType=INTEGER}
+    and catalog = #{catalog,jdbcType=VARCHAR}
+    and name = #{name,jdbcType=VARCHAR}
+    limit 1
+  </select>
+  <insert id="insertSelective" parameterType="org.enthusa.askdata.entity.BiTable"
+          useGeneratedKeys="true" keyProperty="id">
+    insert into bi_table
+    <trim prefix="(" suffix=")" suffixOverrides=",">
+      <if test="dsId != null">
+        ds_id,
+      </if>
+      <if test="catalog != null">
+        catalog,
+      </if>
+      <if test="name != null">
+        name,
+      </if>
+      <if test="brief != null">
+        brief,
+      </if>
+      <if test="remarks != null">
+        remarks,
+      </if>
+      <if test="ownerId != null">
+        owner_id,
+      </if>
+      <if test="updateUserId != null">
+        update_user_id,
+      </if>
+      <if test="status != null">
+        status,
+      </if>
+      <if test="queryCnt != null">
+        query_cnt,
+      </if>
+      <if test="queryUserCnt != null">
+        query_user_cnt,
+      </if>
+      <if test="lastQueryTime != null">
+        last_query_time,
+      </if>
+    </trim>
+    <trim prefix="values (" suffix=")" suffixOverrides=",">
+      <if test="dsId != null">
+        #{dsId,jdbcType=INTEGER},
+      </if>
+      <if test="catalog != null">
+        #{catalog,jdbcType=VARCHAR},
+      </if>
+      <if test="name != null">
+        #{name,jdbcType=VARCHAR},
+      </if>
+      <if test="brief != null">
+        #{brief,jdbcType=VARCHAR},
+      </if>
+      <if test="remarks != null">
+        #{remarks,jdbcType=VARCHAR},
+      </if>
+      <if test="ownerId != null">
+        #{ownerId,jdbcType=INTEGER},
+      </if>
+      <if test="updateUserId != null">
+        #{updateUserId,jdbcType=INTEGER},
+      </if>
+      <if test="status != null">
+        #{status,jdbcType=INTEGER},
+      </if>
+      <if test="queryCnt != null">
+        #{queryCnt,jdbcType=INTEGER},
+      </if>
+      <if test="queryUserCnt != null">
+        #{queryUserCnt,jdbcType=INTEGER},
+      </if>
+      <if test="lastQueryTime != null">
+        #{lastQueryTime,jdbcType=TIMESTAMP},
+      </if>
+    </trim>
+  </insert>
+  <update id="updateByPrimaryKeySelective" parameterType="org.enthusa.askdata.entity.BiTable">
+    update bi_table
+    <set>
+      <if test="dsId != null">
+        ds_id = #{dsId,jdbcType=INTEGER},
+      </if>
+      <if test="catalog != null">
+        catalog = #{catalog,jdbcType=VARCHAR},
+      </if>
+      <if test="name != null">
+        name = #{name,jdbcType=VARCHAR},
+      </if>
+      <if test="brief != null">
+        brief = #{brief,jdbcType=VARCHAR},
+      </if>
+      <if test="remarks != null">
+        remarks = #{remarks,jdbcType=VARCHAR},
+      </if>
+      <if test="ownerId != null">
+        owner_id = #{ownerId,jdbcType=INTEGER},
+      </if>
+      <if test="updateUserId != null">
+        update_user_id = #{updateUserId,jdbcType=INTEGER},
+      </if>
+      <if test="status != null">
+        status = #{status,jdbcType=INTEGER},
+      </if>
+      <if test="queryCnt != null">
+        query_cnt = #{queryCnt,jdbcType=INTEGER},
+      </if>
+      <if test="queryUserCnt != null">
+        query_user_cnt = #{queryUserCnt,jdbcType=INTEGER},
+      </if>
+      <if test="lastQueryTime != null">
+        last_query_time = #{lastQueryTime,jdbcType=TIMESTAMP},
+      </if>
+    </set>
+    where id = #{id,jdbcType=INTEGER}
+  </update>
+  <update id="batchUpdateStatusByPrimaryKeys">
+    update bi_table set status = 1
+    where id in
+    <foreach item='item' collection='list' open='(' separator=',' close=')'>
+      #{item,jdbcType=BIGINT}
+    </foreach>
+  </update>
+  <update id="batchUpdateHotValue">
+    update bi_table
+    <trim prefix="set" suffixOverrides=",">
+      <trim prefix="query_cnt = case" suffix="end,">
+        <foreach collection="list" item="table" index="index">
+          when id=#{table.id} then #{table.queryCnt}
+        </foreach>
+      </trim>
+
+      <trim prefix="query_user_cnt = case" suffix="end,">
+        <foreach collection="list" item="table" index="index">
+          when id=#{table.id} then #{table.queryUserCnt}
+        </foreach>
+      </trim>
+
+      <trim prefix="last_query_time = case" suffix="end,">
+        <foreach collection="list" item="table" index="index">
+          when id=#{table.id} then #{table.lastQueryTime}
+        </foreach>
+      </trim>
+    </trim>
+    where id in
+    <foreach collection="list" index="index" item="table" separator="," open="(" close=")">
+      #{table.id}
+    </foreach>
+  </update>
+</mapper>

src/test/java/org/enthusa/askdata/gpt/ChatGptTest.java

+package org.enthusa.askdata.gpt;
+
+import org.enthusa.askdata.AbstractTest;
+import org.enthusa.askdata.ext.inscode.GptClient;
+import org.enthusa.askdata.ext.inscode.GptRequest;
+import org.junit.Test;
+
+import javax.annotation.Resource;
+
+/**
+ * @author henry
+ * @date 2023/7/1
+ */
+public class ChatGptTest extends AbstractTest {
+    @Resource
+    private GptClient gptClient;
+
+    @Test
+    public void test() throws InterruptedException {
+        GptRequest request = GptRequest.newRequest("Translate natural language to SQL queries.");
+        request.addUserMsg("你能做什么?");
+        request.setApiKey(System.getenv("INSCODE_API_KEY"));
+        System.out.println(gptClient.chatCompletion(request));
+    }
+}

src/test/java/org/enthusa/askdata/task/TaskTest.java

+package org.enthusa.askdata.task;
+
+import org.enthusa.askdata.AbstractTest;
+import org.enthusa.askdata.task.impl.FillMetaDataTask;
+import org.junit.Test;
+
+import javax.annotation.Resource;
+
+/**
+ * @author henry
+ * @date 2023/7/1
+ */
+public class TaskTest extends AbstractTest {
+    @Resource
+    private FillMetaDataTask fillMetaDataTask;
+
+    @Test
+    public void test() {
+        fillMetaDataTask.start();
+    }
+}