Update ShardingSphere version for compatibility verification (#10244)

* Integrate with ShardingSphere 5.3.1-SNAPSHOT.

* Update the version of ShardingSphere in e2e

* Remove providerType when the transaction type is LOCAL.

* Update backend-storage.md & changes.md

docs/en/changes/changes.md

@@ -70,6 +70,7 @@
 * Tweak interval settings of BanyanDB.
 * Support monitoring AWS Cloud EKS.
 * Bump BanyanDB Java client to 0.3.0-rc0.
+* [**Breaking Change**] The supported version of ShardingSphere-Proxy is upgraded from 5.1.2 to 5.3.1. Due to the changes of ShardingSphere's API, versions before 5.3.1 are not compatible.
 
 #### UI
 

docs/en/setup/backend/backend-storage.md

@@ -12,7 +12,7 @@ Natively supported storage:
 - OpenSearch
 - ElasticSearch 6, 7, 8
 - MySQL
-- MySQL-Sharding(Shardingsphere-Proxy 5.1.2)
+- MySQL-Sharding(Shardingsphere-Proxy 5.3.1)
 - TiDB
 - PostgreSQL
 - BanyanDB
@@ -278,7 +278,7 @@ MySQL-Sharding plugin provides the MySQL database sharding and table sharding, t
 leverage [Shardingsphere-Proxy](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-proxy)
 to manage the JDBC between OAP and multi-database instances, and according to the sharding rules do routing to the database and table sharding.
 
-Tested Shardingsphere-Proxy 5.1.2 version, and MySQL Client driver 8.0.13 version is currently available.
+Tested Shardingsphere-Proxy 5.3.1 version, and MySQL Client driver 8.0.13 version is currently available.
 Activate MySQL and Shardingsphere-Proxy as storage, and set storage provider to **mysql-sharding**.
 
 **NOTE:** MySQL driver is NOT allowed in Apache official distribution and source codes.

oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/ShardingSphereTableInstaller.java

@@ -19,6 +19,8 @@
 package org.apache.skywalking.oap.server.storage.plugin.jdbc.shardingsphere;
 
 import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.List;
 import java.util.Set;
 import org.apache.skywalking.oap.server.core.CoreModule;
@@ -30,6 +32,7 @@ import org.apache.skywalking.oap.server.library.client.Client;
 import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException;
 import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
+import org.apache.skywalking.oap.server.storage.plugin.jdbc.TableMetaInfo;
 import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2TableInstaller;
 
 /**
@@ -55,7 +58,7 @@ public class ShardingSphereTableInstaller extends H2TableInstaller {
     @Override
     public boolean isExists(Model model) throws StorageException {
         boolean isRuleExecuted = false;
-        boolean isTableExist = delegatee.isExists(model);
+        boolean isTableExist = isTableExists(model);
         JDBCHikariCPClient jdbcClient = (JDBCHikariCPClient) client;
         ConfigService configService = moduleManager.find(CoreModule.NAME).provider().getService(ConfigService.class);
         int ttl = model.isRecord() ? configService.getRecordDataTTL() : configService.getMetricsDataTTL();
@@ -65,6 +68,20 @@ public class ShardingSphereTableInstaller extends H2TableInstaller {
         return isTableExist && !isRuleExecuted;
     }
 
+    private boolean isTableExists(Model model) throws StorageException {
+        TableMetaInfo.addModel(model);
+        JDBCHikariCPClient jdbcClient = (JDBCHikariCPClient) client;
+        try (Connection conn = jdbcClient.getConnection()) {
+            ResultSet resultSet = jdbcClient.executeQuery(conn, String.format("SHOW LOGICAL TABLES LIKE '%s'", model.getName()));
+            if (resultSet.next()) {
+                return true;
+            }
+        } catch (SQLException | JDBCClientException e) {
+            throw new StorageException(e.getMessage(), e);
+        }
+        return false;
+    }
+
     @Override
     public void start() {
         delegatee.start();

oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/dao/ShardingHistoryDeleteDAO.java

@@ -90,19 +90,16 @@ public class ShardingHistoryDeleteDAO extends JDBCHistoryDeleteDAO {
             }
             List<String> realTables = new ArrayList<>();
             try (Connection connection = client.getConnection()) {
-                ResultSet resultSet = connection.getMetaData()
-                                                .getTables(connection.getCatalog(), null, model.getName() + "_20%", null);
+                ResultSet resultSet = client.executeQuery(connection, String.format("SHOW SINGLE TABLES LIKE '%s'", model.getName() + "_20%"));
                 while (resultSet.next()) {
-                    realTables.add(resultSet.getString("TABLE_NAME"));
+                    realTables.add(resultSet.getString(1));
                 }
 
                 //delete additional tables
                 for (String additionalTable : model.getSqlDBModelExtension().getAdditionalTables().keySet()) {
-                    ResultSet additionalTableRS = connection.getMetaData()
-                                                            .getTables(connection.getCatalog(), null,
-                                                                       additionalTable + "_20%", null);
+                    ResultSet additionalTableRS = client.executeQuery(connection, String.format("SHOW SINGLE TABLES LIKE '%s'", additionalTable + "_20%"));
                     while (additionalTableRS.next()) {
-                        realTables.add(additionalTableRS.getString("TABLE_NAME"));
+                        realTables.add(additionalTableRS.getString(1));
                     }
                 }
             } catch (JDBCClientException | SQLException e) {

oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/shardingsphere/TCITShardingSphere.java

@@ -133,7 +133,7 @@ public class TCITShardingSphere {
     public static Collection<Object[]> versions() {
         return Arrays.asList(new Object[][] {
             {
-                "5.1.2",
+                "26999bd02c3b1613c3508373963334d698c37361",
                 DataSourceType.MYSQL
             }
         });

oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/resources/conf-mysql/server.yaml

@@ -32,31 +32,26 @@
 #      timeToLiveSeconds: 60
 #      maxRetries: 3
 #      operationTimeoutMilliseconds: 500
-#  overwrite: false
 #
-rules:
- - !AUTHORITY
-   users:
-     - root@:root
-   provider:
-     type: ALL_PERMITTED
- - !TRANSACTION
-   defaultType: XA
-   providerType: Atomikos
-   # When the provider type is Narayana, the following properties can be configured or not
-   props:
-     recoveryStoreUrl: jdbc:mysql://127.0.0.1:3306/jbossts
-     recoveryStoreDataSource: com.mysql.jdbc.jdbc2.optional.MysqlDataSource
-     recoveryStoreUser: root
-     recoveryStorePassword: 12345678
- - !SQL_PARSER
-   sqlCommentParseEnabled: true
-   sqlStatementCache:
-     initialCapacity: 2000
-     maximumSize: 65535
-   parseTreeCache:
-     initialCapacity: 128
-     maximumSize: 1024
+#authority:
+#  users:
+#    - user: root@%
+#      password: root
+#  privilege:
+#    type: ALL_PERMITTED
+
+transaction:
+  defaultType: XA
+  providerType: Atomikos
+
+sqlParser:
+  sqlCommentParseEnabled: true
+  sqlStatementCache:
+    initialCapacity: 2000
+    maximumSize: 65535
+  parseTreeCache:
+    initialCapacity: 128
+    maximumSize: 1024
 
 props:
   max-connections-size-per-query: 1
@@ -78,6 +73,6 @@ props:
   sql-federation-enabled: false
     # Available proxy backend driver type: JDBC (default), ExperimentalVertx
   proxy-backend-driver-type: JDBC
-  proxy-mysql-default-version: 5.7.22 # In the absence of schema name, the default version will be used.
+  proxy-mysql-default-version: 8.0.13 # In the absence of schema name, the default version will be used.
   proxy-default-port: 3307 # Proxy default port.
   proxy-netty-backlog: 1024 # Proxy netty backlog.

oap-server/server-storage-plugin/storage-shardingsphere-plugin/src/test/resources/docker-compose-mysql.yml

@@ -17,7 +17,7 @@ version: '2.1'
 
 services:
   sharding-proxy:
-    image: apache/shardingsphere-proxy:${SS_VERSION}
+    image: ghcr.io/apache/shardingsphere-proxy:${SS_VERSION}
     volumes:
       - ./download-mysql.sh:/opt/shardingsphere-proxy/download-mysql.sh
       - ./conf-mysql:/opt/shardingsphere-proxy/conf

test/e2e-v2/script/shardingsphere-proxy/base-compose.yml

@@ -17,7 +17,7 @@ version: '2.1'
 
 services:
   sharding-proxy:
-    image: apache/shardingsphere-proxy:5.1.2
+    image: ghcr.io/apache/shardingsphere-proxy:26999bd02c3b1613c3508373963334d698c37361
     volumes:
       - ./../prepare/setup-oap/download-mysql.sh:/opt/shardingsphere-proxy/download-mysql.sh
     networks:

test/e2e-v2/script/shardingsphere-proxy/conf-mysql/server.yaml

@@ -32,31 +32,25 @@
 #      timeToLiveSeconds: 60
 #      maxRetries: 3
 #      operationTimeoutMilliseconds: 500
-#  overwrite: false
 #
-rules:
- - !AUTHORITY
-   users:
-     - root@:root
-   provider:
-     type: ALL_PERMITTED
- - !TRANSACTION
-   defaultType: XA
-   providerType: Atomikos
-   # When the provider type is Narayana, the following properties can be configured or not
-   props:
-     recoveryStoreUrl: jdbc:mysql://127.0.0.1:3306/jbossts
-     recoveryStoreDataSource: com.mysql.jdbc.jdbc2.optional.MysqlDataSource
-     recoveryStoreUser: root
-     recoveryStorePassword: 12345678
- - !SQL_PARSER
-   sqlCommentParseEnabled: true
-   sqlStatementCache:
-     initialCapacity: 2000
-     maximumSize: 65535
-   parseTreeCache:
-     initialCapacity: 128
-     maximumSize: 1024
+#authority:
+#  users:
+#    - user: root@%
+#      password: root
+#  privilege:
+#    type: ALL_PERMITTED
+
+transaction:
+  defaultType: LOCAL
+
+sqlParser:
+  sqlCommentParseEnabled: true
+  sqlStatementCache:
+    initialCapacity: 2000
+    maximumSize: 65535
+  parseTreeCache:
+    initialCapacity: 128
+    maximumSize: 1024
 
 props:
   max-connections-size-per-query: 1
@@ -67,17 +61,17 @@ props:
   check-table-metadata-enabled: false
   show-process-list-enabled: false
     # Proxy backend query fetch size. A larger value may increase the memory usage of ShardingSphere Proxy.
-    # The default value is -1, which means set the minimum value for different JDBC drivers.
+  # The default value is -1, which means set the minimum value for different JDBC drivers.
   proxy-backend-query-fetch-size: -1
   check-duplicate-table-enabled: false
   proxy-frontend-executor-size: 0 # Proxy frontend executor size. The default value is 0, which means let Netty decide.
     # Available options of proxy backend executor suitable: OLAP(default), OLTP. The OLTP option may reduce time cost of writing packets to client, but it may increase the latency of SQL execution
-    # and block other clients if client connections are more than `proxy-frontend-executor-size`, especially executing slow SQL.
+  # and block other clients if client connections are more than `proxy-frontend-executor-size`, especially executing slow SQL.
   proxy-backend-executor-suitable: OLAP
   proxy-frontend-max-connections: 0 # Less than or equal to 0 means no limitation.
   sql-federation-enabled: false
-    # Available proxy backend driver type: JDBC (default), ExperimentalVertx
+  # Available proxy backend driver type: JDBC (default), ExperimentalVertx
   proxy-backend-driver-type: JDBC
-  proxy-mysql-default-version: 5.7.22 # In the absence of schema name, the default version will be used.
+  proxy-mysql-default-version: 8.0.13 # In the absence of schema name, the default version will be used.
   proxy-default-port: 3307 # Proxy default port.
   proxy-netty-backlog: 1024 # Proxy netty backlog.