From de9963771da0b809d2577423c7b222fb7979306f Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Fri, 8 Nov 2019 11:21:56 +0800 Subject: [PATCH] reset pgsql (#1178) + 1 --- dockerfile/Dockerfile | 76 ++++++++----------- .../conf/application.properties | 26 +++---- .../dao/mapper/CommandMapper.xml | 2 +- .../dao/mapper/ErrorCommandMapper.xml | 2 +- .../dao/mapper/ProcessDefinitionMapper.xml | 29 ++++++- .../dao/mapper/ProcessInstanceMapper.xml | 42 ++++++++++ .../dao/mapper/SessionMapper.xml | 6 ++ .../dao/mapper/TaskInstanceMapper.xml | 1 - .../dao/mapper/UDFUserMapper.xml | 4 +- .../dao/mapper/UdfFuncMapper.xml | 4 +- .../dao/mapper/UserMapper.xml | 12 +-- .../dolphinscheduler/conf/quartz.properties | 10 +-- dockerfile/startup.sh | 45 +++-------- 13 files changed, 144 insertions(+), 115 deletions(-) diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile index 9cedb2e90..e7d793ad9 100644 --- a/dockerfile/Dockerfile +++ b/dockerfile/Dockerfile @@ -1,6 +1,7 @@ FROM ubuntu:18.04 ENV LANG=C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive ARG version ARG tar_version @@ -52,52 +53,46 @@ RUN cd /opt && \ ENV NODE_HOME=/opt/node ENV PATH $PATH:$NODE_HOME/bin -#5,add dolphinscheduler source code to /opt/dolphinscheduler_source +#5,install postgresql +RUN apt-get update && \ + apt-get install -y postgresql postgresql-contrib sudo && \ + sed -i 's/localhost/*/g' /etc/postgresql/10/main/postgresql.conf + +#6,install nginx +RUN apt-get update && \ + apt-get install -y nginx && \ + rm -rf /var/lib/apt/lists/* && \ + echo "\ndaemon off;" >> /etc/nginx/nginx.conf && \ + chown -R www-data:www-data /var/lib/nginx + +#7,install sudo,python,vim,ping and ssh command +RUN apt-get update && \ + apt-get -y install sudo && \ + apt-get -y install python && \ + apt-get -y install vim && \ + apt-get -y install iputils-ping && \ + apt-get -y install net-tools && \ + apt-get -y install openssh-server && \ + apt-get -y install python-pip && \ + pip install kazoo + +#8,add dolphinscheduler source code to /opt/dolphinscheduler_source ADD . /opt/dolphinscheduler_source -#5,backend compilation +#9,backend compilation RUN cd /opt/dolphinscheduler_source && \ mvn -U clean package assembly:assembly -Dmaven.test.skip=true -#6,frontend compilation +#10,frontend compilation RUN chmod -R 777 /opt/dolphinscheduler_source/dolphinscheduler-ui && \ cd /opt/dolphinscheduler_source/dolphinscheduler-ui && \ rm -rf /opt/dolphinscheduler_source/dolphinscheduler-ui/node_modules && \ npm install node-sass --unsafe-perm && \ npm install && \ npm run build -#7,install mysql -RUN echo "deb http://cn.archive.ubuntu.com/ubuntu/ xenial main restricted universe multiverse" >> /etc/apt/sources.list - -RUN echo "mysql-server mysql-server/root_password password root" | debconf-set-selections -RUN echo "mysql-server mysql-server/root_password_again password root" | debconf-set-selections - -RUN apt-get update && \ - apt-get -y install mysql-server-5.7 && \ - mkdir -p /var/lib/mysql && \ - mkdir -p /var/run/mysqld && \ - mkdir -p /var/log/mysql && \ - chown -R mysql:mysql /var/lib/mysql && \ - chown -R mysql:mysql /var/run/mysqld && \ - chown -R mysql:mysql /var/log/mysql - - -# UTF-8 and bind-address -RUN sed -i -e "$ a [client]\n\n[mysql]\n\n[mysqld]" /etc/mysql/my.cnf && \ - sed -i -e "s/\(\[client\]\)/\1\ndefault-character-set = utf8/g" /etc/mysql/my.cnf && \ - sed -i -e "s/\(\[mysql\]\)/\1\ndefault-character-set = utf8/g" /etc/mysql/my.cnf && \ - sed -i -e "s/\(\[mysqld\]\)/\1\ninit_connect='SET NAMES utf8'\ncharacter-set-server = utf8\ncollation-server=utf8_general_ci\nbind-address = 0.0.0.0/g" /etc/mysql/my.cnf - - -#8,install nginx -RUN apt-get update && \ - apt-get install -y nginx && \ - rm -rf /var/lib/apt/lists/* && \ - echo "\ndaemon off;" >> /etc/nginx/nginx.conf && \ - chown -R www-data:www-data /var/lib/nginx -#9,modify dolphinscheduler configuration file +#11,modify dolphinscheduler configuration file #backend configuration RUN mkdir -p /opt/dolphinscheduler && \ tar -zxvf /opt/dolphinscheduler_source/target/dolphinscheduler-${tar_version}.tar.gz -C /opt/dolphinscheduler && \ @@ -106,22 +101,11 @@ ADD ./dockerfile/conf/dolphinscheduler/conf /opt/dolphinscheduler/conf #frontend nginx configuration ADD ./dockerfile/conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d -#10,open port +#12,open port EXPOSE 2181 2888 3888 3306 80 12345 8888 -#11,install sudo,python,vim,ping and ssh command -RUN apt-get update && \ - apt-get -y install sudo && \ - apt-get -y install python && \ - apt-get -y install vim && \ - apt-get -y install iputils-ping && \ - apt-get -y install net-tools && \ - apt-get -y install openssh-server && \ - apt-get -y install python-pip && \ - pip install kazoo - COPY ./dockerfile/startup.sh /root/startup.sh -#12,modify permissions and set soft links +#13,modify permissions and set soft links RUN chmod +x /root/startup.sh && \ chmod +x /opt/dolphinscheduler/script/create-dolphinscheduler.sh && \ chmod +x /opt/zookeeper/bin/zkServer.sh && \ diff --git a/dockerfile/conf/dolphinscheduler/conf/application.properties b/dockerfile/conf/dolphinscheduler/conf/application.properties index af8c449f9..e0add2565 100644 --- a/dockerfile/conf/dolphinscheduler/conf/application.properties +++ b/dockerfile/conf/dolphinscheduler/conf/application.properties @@ -1,10 +1,8 @@ -# mysql -# url=jdbc:postgresql://192.168.220.154:5432/dolphinscheduler # base spring data source configuration spring.datasource.type=com.alibaba.druid.pool.DruidDataSource -#spring.datasource.driver-class-name=org.postgresql.Driver -spring.datasource.driver-class-name=com.mysql.jdbc.Driver -spring.datasource.url=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8 +# postgre +spring.datasource.driver-class-name=org.postgresql.Driver +spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler spring.datasource.username=root spring.datasource.password=root@123 @@ -29,7 +27,7 @@ spring.datasource.timeBetweenConnectErrorMillis=60000 spring.datasource.minEvictableIdleTimeMillis=300000 #the SQL used to check whether the connection is valid requires a query statement. If validation Query is null, testOnBorrow, testOnReturn, and testWhileIdle will not work. -spring.datasource.validationQuery=SELECT 1 FROM DUAL +spring.datasource.validationQuery=SELECT 1 #check whether the connection is valid for timeout, in seconds spring.datasource.validationQueryTimeout=3 @@ -57,21 +55,21 @@ mybatis-plus.mapper-locations=classpath*:/org.apache.dolphinscheduler.dao.mapper mybatis-plus.typeEnumsPackage=org.apache.dolphinscheduler.*.enums -#实体扫描，多个package用逗号或者分号分隔 +#Entity scan, where multiple packages are separated by a comma or semicolon mybatis-plus.typeAliasesPackage=org.apache.dolphinscheduler.dao.entity -#主键类型 AUTO:"数据库ID自增", INPUT:"用户输入ID", ID_WORKER:"全局唯一ID (数字类型唯一ID)", UUID:"全局唯一ID UUID"; +#Primary key type AUTO:" database ID AUTO ", INPUT:" user INPUT ID", ID_WORKER:" global unique ID (numeric type unique ID)", UUID:" global unique ID UUID"; mybatis-plus.global-config.db-config.id-type=AUTO -#字段策略 IGNORED:"忽略判断",NOT_NULL:"非 NULL 判断"),NOT_EMPTY:"非空判断" +#Field policy IGNORED:" ignore judgment ",NOT_NULL:" not NULL judgment "),NOT_EMPTY:" not NULL judgment" mybatis-plus.global-config.db-config.field-strategy=NOT_NULL -#驼峰下划线转换 +#The hump underline is converted mybatis-plus.global-config.db-config.column-underline=true mybatis-plus.global-config.db-config.logic-delete-value=-1 mybatis-plus.global-config.db-config.logic-not-delete-value=0 mybatis-plus.global-config.db-config.banner=false -#原生配置 +#The original configuration mybatis-plus.configuration.map-underscore-to-camel-case=true mybatis-plus.configuration.cache-enabled=false mybatis-plus.configuration.call-setters-on-nulls=true @@ -80,9 +78,9 @@ mybatis-plus.configuration.jdbc-type-for-null=null # data quality analysis is not currently in use. please ignore the following configuration # task record flag task.record.flag=false -task.record.datasource.url=jdbc:mysql://127.0.0.1:3306/etl?characterEncoding=UTF-8 -task.record.datasource.username=xx -task.record.datasource.password=xx +task.record.datasource.url=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler +task.record.datasource.username=root +task.record.datasource.password=root@123 # Logger Config logging.level.org.apache.dolphinscheduler.dao=debug diff --git a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml index e3ab6b23c..370b0119b 100644 --- a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml +++ b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/CommandMapper.xml @@ -9,7 +9,7 @@ limit 1 - select cmd.command_type as commandType, count(1) as count + select cmd.command_type as command_type, count(1) as count from t_ds_error_command cmd, t_ds_process_definition process where cmd.process_definition_id = process.id diff --git a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml index ed53f8154..18447de26 100644 --- a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml +++ b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml @@ -12,10 +12,14 @@ and pd.name = #{processDefinitionName} + + - - + \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml index c1b7f909f..75dc44888 100644 --- a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml +++ b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/ProcessInstanceMapper.xml @@ -19,6 +19,35 @@ order by id asc + + + + + select t.state, count(0) as count from t_ds_process_instance t diff --git a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/SessionMapper.xml b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/SessionMapper.xml index 7647809e8..48297cf94 100644 --- a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/SessionMapper.xml +++ b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/SessionMapper.xml @@ -6,4 +6,10 @@ from t_ds_session where user_id = #{userId} + + \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml index 77290dee7..2acfd6eb0 100644 --- a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml +++ b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml @@ -50,7 +50,6 @@ #{i} - and t.flag = 1 and t.start_time > #{startTime} and t.start_time #{endTime} diff --git a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.xml b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.xml index afa35945e..c42641055 100644 --- a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.xml +++ b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UDFUserMapper.xml @@ -3,10 +3,10 @@ delete from t_ds_relation_udfs_user - where user_id=#{userId} + where user_id = #{userId} delete from t_ds_relation_udfs_user - where udf_id=#{udfFuncId} + where udf_id = #{udfFuncId} \ No newline at end of file diff --git a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index 0fb496f4a..cf89023ca 100644 --- a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -12,7 +12,7 @@ - and func_name = #{funcName} + and func_name = #{funcNames} order by id asc @@ -21,7 +21,7 @@ from t_ds_udfs where 1=1 - and name like concat('%', #{searchVal}, '%') + and func_name like concat('%', #{searchVal}, '%') and id in ( diff --git a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml index 55562ce6e..f111e91d8 100644 --- a/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml +++ b/dockerfile/conf/dolphinscheduler/conf/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml @@ -1,7 +1,6 @@ - - - - + -