From 0ff06897f73f03d30fb5717d71ce5b6f01270853 Mon Sep 17 00:00:00 2001
From: shimingxy <shimingxy@163.com>
Date: Tue, 9 Apr 2019 22:31:11 +0800
Subject: [PATCH] README

---
 README.md | 215 ++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 200 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 37a70c0..dabf54d 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-#Blazer/数据传播者
+# Blazer/数据传播者
 Blazer意为**传播者**，主要实现不同数据库之间的数据同步，导出数据库表结构创建语句。
 
 1. 导出数据库表数据到txt/CSV文件（多线程）或者excel(.xlsx/.xls)；
@@ -118,7 +118,7 @@ ${JAVA_EXEC} ${JAVA_OPTS} -classpath ${JAVA_CLASSPATH} ${JAVA_MAINCLASS} --confi
 echo run finished
 ```
 
-实现Oracle到greenplum数据同步
+###### 实现Oracle到greenplum数据同步pipeLine
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
 <beans 	xmlns="http://www.springframework.org/schema/beans"
@@ -133,7 +133,7 @@ echo run finished
 	        http://www.springframework.org/schema/util     
 	        http://www.springframework.org/schema/util/spring-util.xsd">
  	   
-    <!-- Application properties configs  -->
+	<!-- Application properties configs  应用程序属性配置文件-->
 	<bean id="propertySourcesPlaceholderConfigurer" class="org.springframework.context.support.PropertySourcesPlaceholderConfigurer">
 	  <property name="locations">
 	    <list>
@@ -148,9 +148,7 @@ echo run finished
  	
  	<!-- Datastore configuration /数据源配置 -->
  	<import resource="database.xml"/>
- 	
- 	<!-- FULL      在插入前进行删除,默认    -->
-	<!-- INCREMENT 先清除条件相关数据，然后按照条件进行增量插入 -->
+
  	<bean id="transData" class="com.blazer.trans.TransData">
  		<!--源数据库-->
 		<property name="sourceDataSource" ref="datasource_oracle"/>
@@ -160,35 +158,222 @@ echo run finished
  		<property name="targetDataSource" ref="datasource_greenplum"/>
  		<property name="toUrl" value="${datasource_greenplum.url}"/>
  		<property name="toUser" value="${datasource_greenplum.username}"/> 
- 		
-		<!--全量-->
+		<!-- FULL      在插入前进行删除,默认    -->
+		<!-- INCREMENT 先清除条件相关数据，然后按照条件进行增量插入 -->
  		<property name="transType" value="FULL"/> 
  		<property name="tablesList" > 
  			<util:list  list-class="java.util.ArrayList">
 			 	<bean class="com.db.TableDescribe">
 					<!--源表名-->
- 					<property name="tableName" value="SIGNAL"/> 
+ 					<property name="tableName" value="STUDENT"/> 
 					<!--目标表名-->
- 					<property name="targetTableName" value="SIGNAL"/> 
+ 					<property name="targetTableName" value="STUDENT"/> 
 					<!--筛选条件-->
-					<property name="whereSqlString" value="name='TEST'"/>
+					<property name="whereSqlString" value="CLASSES='2020'"/>
  				</bean>
  				<bean class="com.db.TableDescribe">
  					<property name="tableName" value="CALENDAR"/> 
  					<property name="targetTableName" value="M_CALENDAR"/> 
  				</bean>
-
- 				
  			</util:list>
  		</property>
  	</bean>
  	
- 
-
  	<!-- 配置执行的任务列表  -->
  	<util:list id="pipeLineTask" list-class="java.util.ArrayList">
  		<ref bean="transData"/>
  	</util:list>
+</beans>
+```
+######数据库数据导出到文件(csv,xlsx,xls)pipeLine
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<beans 	xmlns="http://www.springframework.org/schema/beans"
+		xmlns:context="http://www.springframework.org/schema/context"
+		xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		xmlns:util="http://www.springframework.org/schema/util"
+		xsi:schemaLocation="
+	        http://www.springframework.org/schema/beans     
+	        http://www.springframework.org/schema/beans/spring-beans.xsd
+	        http://www.springframework.org/schema/context 
+	        http://www.springframework.org/schema/context/spring-context.xsd
+	        http://www.springframework.org/schema/util     
+	        http://www.springframework.org/schema/util/spring-util.xsd">
+			
+ 	<!-- Application properties configs  应用程序属性配置文件-->
+	<bean id="propertySourcesPlaceholderConfigurer" class="org.springframework.context.support.PropertySourcesPlaceholderConfigurer">
+	  <property name="locations">
+	    <list>
+	   	  	<value>/pipeline/config/applicationConfig.properties</value>
+	    </list>
+	  </property>
+	  <property name="ignoreUnresolvablePlaceholders" value="true"/>
+	</bean>
+ 	
+ 	<!-- Datastore configuration /数据源配置 -->
+ 	<import resource="database.xml"/>
  	
+	<!--基本配置-->
+ 	<bean id="exportBasicConfigure" class="com.blazer.export.file.BasicConfigure">
+ 		<property name="sourceDataSource" ref="datasource_oracle"/>
+		<property name="fromUrl" value="${datasource_oracle.url}"/>
+ 		<property name="fromUser" value="${datasource_oracle.username}"/> 
+		<!--每次写条数-->
+ 		<property name="commitNumber" value="400"/>
+		<!--导出线程-->
+		<property name="threadSize" value="1"/>
+		<!--导出文件格式 csv/xlsx/xls-->
+		<property name="fileType" value="csv"/>
+		<!--导出文件路径-->
+ 		<property name="exportFilePath" value="D:/dmp/"/>
+		<!--导出文件后缀 csv(.txt,.csv)/xlsx(.xlsx)/xls(.xls) -->
+ 		<property name="fileNameSuffix" value=".txt"/>
+		<!--导出文件字段分割-->
+ 		<property name="terminatedString" value="|+|"/>
+ 	</bean>	
+ 	<!--导出基本配置-->	
+ 	<bean id="transDataExport_STUD" class="com.blazer.export.file.TransDataExport">
+ 		<property name="sourceDataSource" ref="datasource_oracle"/>
+		<property name="fromUrl" value="${datasource_oracle.url}"/>
+ 		<property name="fromUser" value="${datasource_oracle.username}"/> 
+		<!--导出表名-->
+		<property name="tableName" value="STUDENT"/>
+		<!--导出文件名，其中{yyyyMMdd}为日期格式-->
+		<property name="outFileName" value="STUDENT_{yyyyMMdd}_000_000"/>
+		<!--每次写条数-->
+ 		<property name="commitNumber" value="400"/>
+		<!--导出线程-->
+		<property name="threadSize" value="1"/>
+		<!--导出文件格式 csv/xlsx/xls-->
+		<property name="fileType" value="csv"/>
+		<!--导出文件路径-->
+ 		<property name="exportFilePath" value="D:/dmp/"/>
+		<!--导出文件后缀 csv(.txt,.csv)/xlsx(.xlsx)/xls(.xls) -->
+ 		<property name="fileNameSuffix" value=".txt"/>
+		<!--导出文件字段分割-->
+ 		<property name="terminatedString" value="|+|"/>
+		<!--导出表数据过滤条件-->
+		<!--
+		<property name="whereSqlString" value="where length(body)&gt;10000 and length(body)&lt;50000" and FLAG='_EXPORT_ETL_DATE_'/>-->
+		<property name="whereSqlString" value=""/>
+		<!--导出字段，如果是*则全表导出-->
+ 		<property name="selectFieldsString" value="
+			STUD_NAME           ,
+			STUD_SEX         
+		"
+		/>
+ 	</bean>
+
+ 	<!-- 配置执行的任务列表  -->
+ 	<util:list id="pipeLineTask" list-class="java.util.ArrayList">
+		<ref bean="transDataExport_STUD"/>
+ 	</util:list>
 </beans>
 ```
+###### 数据文件导入到数据库
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<beans 	xmlns="http://www.springframework.org/schema/beans"
+		xmlns:context="http://www.springframework.org/schema/context"
+		xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+		xmlns:util="http://www.springframework.org/schema/util"
+		xsi:schemaLocation="
+	        http://www.springframework.org/schema/beans     
+	        http://www.springframework.org/schema/beans/spring-beans.xsd
+	        http://www.springframework.org/schema/context 
+	        http://www.springframework.org/schema/context/spring-context.xsd
+	        http://www.springframework.org/schema/util     
+	        http://www.springframework.org/schema/util/spring-util.xsd">
+
+ 	<!-- Application properties configs  应用程序属性配置文件-->
+	<bean id="propertySourcesPlaceholderConfigurer" class="org.springframework.context.support.PropertySourcesPlaceholderConfigurer">
+	  <property name="locations">
+	    <list>
+	   	  	<value>/pipeline/config/applicationConfig.properties</value>
+	    </list>
+	  </property>
+	  <property name="ignoreUnresolvablePlaceholders" value="true"/>
+	</bean>
+ 	
+ 	<!-- Datastore configuration /数据源配置 -->
+ 	<import resource="database.xml"/>
+ 	
+ 	<bean id="transDataLoad_STUD" class="com.blazer.load.file.runner.TransDataLoadFile">
+		<!--导入数据源-->
+ 		<property name="sourceDataSource" ref="datasource_oracle"/>
+		<!--每次提交数据量-->
+ 		<property name="commitNumber" value="100"/>
+		<!--字段最长限制-->
+ 		<property name="limitTextSize" value="0"/>
+		<!--导入目标表名-->
+ 		<property name="tableName" value="STUDENT"/>
+		<!--导入文件名-->
+ 		<property name="loadFileName" value="student_2020"/>
+		<!--导入文件扩展名类型 csv/xlsx/xls-->
+ 		<property name="fileType" value="xls"/>
+		<!--导入文件路径-->
+ 		<property name="loadFilePath" value="D:/dmp/"/>
+		<!--是否跳过首行 true是跳过，false不跳过，默认为false-->
+ 		<property name="skipFirstRow" value="true"/>
+ 		<!--导入文件后缀-->
+ 		<property name="fileNameSuffix" value=".xls"/>
+		<!--导入csv类型文件分隔符-->
+ 		<property name="terminatedString" value="|+|"/>
+		<!--导入字段列表-->
+ 		<property name="listTableColumns" > 
+ 			<util:list  list-class="java.util.ArrayList">
+				<bean class="com.db.TableColumns">
+					<!--导入字段名-->
+					<property name="columnName" value="STUD_NAME"/>
+					<!--导入字段类型-->
+ 					<property name="dataType" value="VARCHAR"/>
+					<!--导入字段是否跳过-->
+					<property name="skip" value="false"/>
+					<!--导入字段为固定值-->
+					<property name="fixed" value="false"/>
+					<!--导入字段默认值，配合fixed使用-->
+					<property name="defaultValue" value=""/>
+				</bean>
+				<bean class="com.db.TableColumns">
+					<property name="columnName" value="STUD_SEX"/>
+ 					<property name="dataType" value="VARCHAR"/>
+					<!--导入字段基于javascript转换脚本-->
+					<property name="convert" value="
+						if(dataValue=='M'){
+							returnValue='男';
+						}else if(columns[1]=='男'){
+							returnValue='男';
+						}
+						if(dataValue=='F'){
+							returnValue='女';
+						}
+					"/>
+				</bean>
+		 	</util:list>
+ 		</property>
+ 		
+ 	</bean>
+ 	
+ 	<bean id="transDataLoad" class="com.blazer.load.file.TransDataLoad">
+		<!--默认数据源-->
+ 		<property name="sourceDataSource" ref="datasource_oracle"/>
+		<property name="fromUrl" value="${datasource_oracle.url}"/>
+ 		<property name="fromUser" value="${datasource_oracle.username}"/> 
+		<!--默认导入线程数-->
+ 		<property name="threadSize" value="10"/>
+		<!--默认导入文件路径-->
+ 		<property name="loadFilePath" value="D:/dmp/"/>
+		<!--导入任务列表-->
+ 		<property name="transDataLoadFileList" > 
+ 			<util:list  list-class="java.util.ArrayList">
+				<ref bean="transDataLoad_STUD"/>
+		 	</util:list>
+ 		</property>
+ 	</bean>
+ 
+ 	<!-- 配置执行的任务列表  -->
+ 	<util:list id="pipeLineTask" list-class="java.util.ArrayList">
+		<ref bean="transDataLoad"/>
+ 	</util:list>
+</beans>
+```
\ No newline at end of file
-- 
GitLab