From abc1e0a1ffbc907ffb1ce56c504dfbc6bf0357d6 Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Mon, 29 Jul 2019 14:05:41 +0200 Subject: [PATCH] [FLINK-13399][table] Create two separate table uber jars for old and blink planners --- flink-dist/pom.xml | 7 + flink-dist/src/main/assemblies/bin.xml | 15 +++ flink-dist/src/main/assemblies/opt.xml | 8 -- .../flink-bin/bin/pyflink-gateway-server.sh | 2 +- .../flink-stream-sql-test/pom.xml | 6 - .../test-scripts/test_streaming_sql.sh | 3 - .../start-script/start-scala-shell.sh | 7 - flink-table/flink-sql-client/pom.xml | 12 -- flink-table/flink-table-planner/pom.xml | 2 +- flink-table/flink-table-runtime-blink/pom.xml | 7 + flink-table/flink-table-uber-blink/pom.xml | 122 ++++++++++++++++++ flink-table/flink-table-uber/pom.xml | 6 + flink-table/pom.xml | 1 + tools/travis_controller.sh | 2 +- 14 files changed, 161 insertions(+), 39 deletions(-) create mode 100644 flink-table/flink-table-uber-blink/pom.xml diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml index d14cc632c93..1e80eaad2f0 100644 --- a/flink-dist/pom.xml +++ b/flink-dist/pom.xml @@ -299,6 +299,13 @@ under the License. provided + + org.apache.flink + flink-table-uber-blink_${scala.binary.version} + ${project.version} + provided + + org.apache.flink flink-sql-client_${scala.binary.version} diff --git a/flink-dist/src/main/assemblies/bin.xml b/flink-dist/src/main/assemblies/bin.xml index 869cffa3418..72891530a1a 100644 --- a/flink-dist/src/main/assemblies/bin.xml +++ b/flink-dist/src/main/assemblies/bin.xml @@ -52,6 +52,21 @@ under the License. 0644 + + + ../flink-table/flink-table-uber/target/flink-table-uber_${scala.binary.version}-${project.version}.jar + lib/ + flink-table_${scala.binary.version}-${project.version}.jar + 0644 + + + + ../flink-table/flink-table-uber-blink/target/flink-table-uber-blink_${scala.binary.version}-${project.version}.jar + lib/ + flink-table-blink_${scala.binary.version}-${project.version}.jar + 0644 + + src/main/resources/flink-conf.yaml diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml index b2f989a06ba..e2dbd6587f8 100644 --- a/flink-dist/src/main/assemblies/opt.xml +++ b/flink-dist/src/main/assemblies/opt.xml @@ -59,14 +59,6 @@ 0644 - - - ../flink-table/flink-table-uber/target/flink-table-uber_${scala.binary.version}-${project.version}.jar - opt/ - flink-table_${scala.binary.version}-${project.version}.jar - 0644 - - ../flink-table/flink-sql-client/target/flink-sql-client_${scala.binary.version}-${project.version}.jar diff --git a/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh b/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh index ce79f1affc1..16fe6b3f95b 100644 --- a/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh +++ b/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh @@ -50,7 +50,7 @@ done log=$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-python-$HOSTNAME.log log_setting=(-Dlog.file="$log" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/log4j-cli.properties -Dlogback.configurationFile=file:"$FLINK_CONF_DIR"/logback.xml) -TABLE_JAR_PATH=`echo "$FLINK_HOME"/opt/flink-table*.jar` +TABLE_JAR_PATH=`echo "$FLINK_HOME"/lib/flink-table*.jar` PYTHON_JAR_PATH=`echo "$FLINK_HOME"/opt/flink-python*.jar` FLINK_TEST_CLASSPATH="" diff --git a/flink-end-to-end-tests/flink-stream-sql-test/pom.xml b/flink-end-to-end-tests/flink-stream-sql-test/pom.xml index f478867849c..53c947df0c4 100644 --- a/flink-end-to-end-tests/flink-stream-sql-test/pom.xml +++ b/flink-end-to-end-tests/flink-stream-sql-test/pom.xml @@ -49,12 +49,6 @@ ${project.version} provided - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${project.version} - provided - diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh b/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh index afc9bb39659..bfae7d5cc21 100755 --- a/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh +++ b/flink-end-to-end-tests/test-scripts/test_streaming_sql.sh @@ -21,9 +21,6 @@ source "$(dirname "$0")"/common.sh TEST_PROGRAM_JAR=${END_TO_END_DIR}/flink-stream-sql-test/target/StreamSQLTestProgram.jar -# copy flink-table jar into lib folder -add_optional_lib "table" - start_cluster $FLINK_DIR/bin/taskmanager.sh start $FLINK_DIR/bin/taskmanager.sh start diff --git a/flink-scala-shell/start-script/start-scala-shell.sh b/flink-scala-shell/start-script/start-scala-shell.sh index 96ca805742b..b6da81af729 100644 --- a/flink-scala-shell/start-script/start-scala-shell.sh +++ b/flink-scala-shell/start-script/start-scala-shell.sh @@ -52,13 +52,6 @@ bin=`cd "$bin"; pwd` FLINK_CLASSPATH=`constructFlinkClassPath` -# Append flink-table jar into class path -opt=`dirname "$0"` -opt=`cd "$opt"/../opt; pwd` -FLINK_TABLE_LIB_PATH=$opt/`ls $opt|grep flink-table_*` -FLINK_CLASSPATH=$FLINK_CLASSPATH:$FLINK_TABLE_LIB_PATH - - # https://issues.scala-lang.org/browse/SI-6502, cant load external jars interactively # in scala shell since 2.10, has to be done at startup # checks arguments for additional classpath and adds it to the "standard classpath" diff --git a/flink-table/flink-sql-client/pom.xml b/flink-table/flink-sql-client/pom.xml index 8e640ffeec9..f725ff0dc34 100644 --- a/flink-table/flink-sql-client/pom.xml +++ b/flink-table/flink-sql-client/pom.xml @@ -59,12 +59,6 @@ under the License. ${project.version} - - org.apache.flink - flink-cep_${scala.binary.version} - ${project.version} - - org.apache.flink @@ -515,12 +509,6 @@ under the License. - org.apache.flink:flink-table-common - org.apache.flink:flink-table-api-java - org.apache.flink:flink-table-api-java-bridge_${scala.binary.version} - org.apache.flink:flink-table-planner_${scala.binary.version} - org.apache.flink:flink-cep_${scala.binary.version} - org.apache.flink:flink-sql-parser org.jline:* diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml index 14aa9662247..c1f8a1fb649 100644 --- a/flink-table/flink-table-planner/pom.xml +++ b/flink-table/flink-table-planner/pom.xml @@ -130,7 +130,6 @@ under the License. provided - org.apache.flink flink-cep_${scala.binary.version} @@ -138,6 +137,7 @@ under the License. provided + org.codehaus.janino janino diff --git a/flink-table/flink-table-runtime-blink/pom.xml b/flink-table/flink-table-runtime-blink/pom.xml index f16d5a295c9..52891a92ac3 100644 --- a/flink-table/flink-table-runtime-blink/pom.xml +++ b/flink-table/flink-table-runtime-blink/pom.xml @@ -170,6 +170,13 @@ under the License. org.lz4:lz4-java + + + + net.jpountz + org.apache.flink.table.shaded.net.jpountz + + diff --git a/flink-table/flink-table-uber-blink/pom.xml b/flink-table/flink-table-uber-blink/pom.xml new file mode 100644 index 00000000000..ffd40a4e95c --- /dev/null +++ b/flink-table/flink-table-uber-blink/pom.xml @@ -0,0 +1,122 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-table + 1.10-SNAPSHOT + .. + + + flink-table-uber-blink_${scala.binary.version} + flink-table-uber-blink + + This module contains the entire Table/SQL distribution for writing table programs + within the table ecosystem or between other Flink APIs. This module uses the Blink planner + for generating optimized runnable plan from relational query. Users can either use the + Scala or Java programming language. + + + jar + + + + org.apache.flink + flink-table-common + ${project.version} + + + org.apache.flink + flink-sql-parser + ${project.version} + + + org.apache.flink + flink-table-api-java + ${project.version} + + + org.apache.flink + flink-table-api-scala_${scala.binary.version} + ${project.version} + + + org.apache.flink + flink-table-api-java-bridge_${scala.binary.version} + ${project.version} + + + org.apache.flink + flink-table-api-scala-bridge_${scala.binary.version} + ${project.version} + + + org.apache.flink + flink-table-planner-blink_${scala.binary.version} + ${project.version} + + + org.apache.flink + flink-table-runtime-blink_${scala.binary.version} + ${project.version} + + + org.apache.flink + flink-cep_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade-flink + package + + shade + + + + + org.apache.flink:flink-table-common + org.apache.flink:flink-sql-parser + org.apache.flink:flink-table-api-java + org.apache.flink:flink-table-api-scala_${scala.binary.version} + org.apache.flink:flink-table-api-java-bridge_${scala.binary.version} + org.apache.flink:flink-table-api-scala-bridge_${scala.binary.version} + org.apache.flink:flink-table-planner-blink_${scala.binary.version} + org.apache.flink:flink-table-runtime-blink_${scala.binary.version} + org.apache.flink:flink-cep_${scala.binary.version} + + + + + + + + + diff --git a/flink-table/flink-table-uber/pom.xml b/flink-table/flink-table-uber/pom.xml index c5f61f019e2..aa2c43462dc 100644 --- a/flink-table/flink-table-uber/pom.xml +++ b/flink-table/flink-table-uber/pom.xml @@ -73,6 +73,11 @@ under the License. flink-table-planner_${scala.binary.version} ${project.version} + + org.apache.flink + flink-cep_${scala.binary.version} + ${project.version} + @@ -98,6 +103,7 @@ under the License. org.apache.flink:flink-table-api-java-bridge_${scala.binary.version} org.apache.flink:flink-table-api-scala-bridge_${scala.binary.version} org.apache.flink:flink-table-planner_${scala.binary.version} + org.apache.flink:flink-cep_${scala.binary.version} diff --git a/flink-table/pom.xml b/flink-table/pom.xml index a8e2724f0d1..1e479b35239 100644 --- a/flink-table/pom.xml +++ b/flink-table/pom.xml @@ -42,6 +42,7 @@ under the License. flink-table-planner-blink flink-table-runtime-blink flink-table-uber + flink-table-uber-blink flink-sql-client flink-sql-parser diff --git a/tools/travis_controller.sh b/tools/travis_controller.sh index 211551da683..3f3f5d86c68 100755 --- a/tools/travis_controller.sh +++ b/tools/travis_controller.sh @@ -146,7 +146,7 @@ if [ $STAGE == "$STAGE_COMPILE" ]; then ! -path "$CACHE_FLINK_DIR/flink-runtime/target/flink-runtime*tests.jar" \ ! -path "$CACHE_FLINK_DIR/flink-streaming-java/target/flink-streaming-java*tests.jar" \ ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-dist*.jar" \ - ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/opt/flink-table*.jar" \ + ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/lib/flink-table*.jar" \ ! -path "$CACHE_FLINK_DIR/flink-dist/target/flink-*-bin/flink-*/opt/flink-python*.jar" \ ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-elasticsearch-base/target/flink-*.jar" \ ! -path "$CACHE_FLINK_DIR/flink-connectors/flink-connector-kafka-base/target/flink-*.jar" \ -- GitLab