增加dumpData,restoreData 中attachStorage配置

o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java

@@ -17,35 +17,31 @@ public class DumpRestoreData extends ConfigObject {
 		return new DumpRestoreData();
 	}
 
-	public static final String TYPE_FULL = "full";
-	public static final String TYPE_LITE = "lite";
+	public static final String MODE_FULL = "full";
+	public static final String MODE_LITE = "lite";
 	public static final String RESTOREOVERRIDE_CLEAN = "clean";
 	public static final String RESTOREOVERRIDE_SKIPEXISTED = "skipExisted";
 
-	public static final String DEFAULT_TYPE = TYPE_LITE;
 	public static final Boolean DEFAULT_PARALLEL = true;
 	public static final Boolean DEFAULT_REDISTRIBUTE = true;
 	public static final Boolean DEFAULT_EXCEPTIONINVALIDSTORAGE = true;
+	public static final Boolean DEFAULT_ATTACHSTORAGE = true;
 	public static final String DEFAULT_ITEMCATEGORY = "";
 
 	public DumpRestoreData() {
-		this.enable = false;
 		this.includes = new ArrayList<>();
 		this.excludes = new ArrayList<>();
-		this.mode = DEFAULT_TYPE;
+		this.mode = MODE_LITE;
 		this.parallel = DEFAULT_PARALLEL;
-		this.redistribute = DEFAULT_REDISTRIBUTE;
+		this.attachStorage = DEFAULT_ATTACHSTORAGE;
 		this.exceptionInvalidStorage = DEFAULT_EXCEPTIONINVALIDSTORAGE;
 		this.itemCategory = DEFAULT_ITEMCATEGORY;
 	}
 
-	@FieldDescribe("是否启用.")
-	private Boolean enable;
-
-	@FieldDescribe("导出导入包含对象,可以使用通配符*.")
+	@FieldDescribe("导出导入包含对象,可以使用通配符*,如仅导出待办数据:com.x.processplatform.core.entity.content.Task.")
 	private List<String> includes;
 
-	@FieldDescribe("导出导入排除对象,可以使用通配符*.")
+	@FieldDescribe("导出导入排除对象,可以使用通配符*,如不导出流程实例数据com.x.processplatform.core.entity.content.*")
 	private List<String> excludes;
 
 	@FieldDescribe("导出数据模式,lite|full,默认使用lite")
@@ -54,31 +50,41 @@ public class DumpRestoreData extends ConfigObject {
 	@FieldDescribe("使用并行导出,默认true")
 	private Boolean parallel;
 
-	@FieldDescribe("是否进行重新分布.")
-	private Boolean redistribute;
-
 	@FieldDescribe("无法获取storage是否升起错误.")
 	private Boolean exceptionInvalidStorage;
 
-	@FieldDescribe("无法获取storage是否升起错误.")
-	private Boolean storageXXXXXXXXXXXXXXXX;
-	
+	@FieldDescribe("是否同时导入导出storage中存储的文件.")
+	private Boolean attachStorage;
+
+	@FieldDescribe("是否进行重新分布.")
+	private Boolean redistributeStorage;
+
+	public Boolean getRedistributeStorage() {
+		return redistributeStorage;
+	}
+
+	public void setRedistributeStorage(Boolean redistributeStorage) {
+		this.redistributeStorage = redistributeStorage;
+	}
+
+	public Boolean getAttachStorage() {
+		return attachStorage;
+	}
+
+	public void setAttachStorage(Boolean attachStorage) {
+		this.attachStorage = attachStorage;
+	}
+
 	@FieldDescribe("数据导入方式,clean:清空重新导入,skipExisted:如果有相同id的数据跳过.默认方式为clean.")
 	private String restoreOverride;
 
 	@FieldDescribe("对于com.x.query.core.entity.Item的itemCategory进行单独过滤,可选值pp, cms, bbs, pp_dict.")
 	private String itemCategory;
-	
-	
 
 	public String getItemCategory() {
 		return this.itemCategory;
 	}
 
-	public Boolean getRedistribute() {
-		return BooleanUtils.isNotFalse(redistribute);
-	}
-
 	public Boolean getExceptionInvalidStorage() {
 		return BooleanUtils.isNotFalse(exceptionInvalidStorage);
 	}
@@ -88,11 +94,7 @@ public class DumpRestoreData extends ConfigObject {
 	}
 
 	public String getMode() {
-		return StringUtils.equals(TYPE_FULL, mode) ? TYPE_FULL : TYPE_LITE;
-	}
-
-	public Boolean getEnable() {
-		return BooleanUtils.isTrue(this.enable);
+		return StringUtils.equals(MODE_FULL, mode) ? MODE_FULL : MODE_LITE;
 	}
 
 	public List<String> getIncludes() {
@@ -123,18 +125,10 @@ public class DumpRestoreData extends ConfigObject {
 		this.parallel = parallel;
 	}
 
-	public void setRedistribute(Boolean redistribute) {
-		this.redistribute = redistribute;
-	}
-
 	public void setExceptionInvalidStorage(Boolean exceptionInvalidStorage) {
 		this.exceptionInvalidStorage = exceptionInvalidStorage;
 	}
 
-	public void setEnable(Boolean enable) {
-		this.enable = enable;
-	}
-
 	public void setMode(String mode) {
 		this.mode = mode;
 	}

o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java

@@ -10,7 +10,9 @@ import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import javax.persistence.EntityManager;
@@ -36,6 +38,7 @@ import com.x.base.core.entity.dataitem.DataItem;
 import com.x.base.core.entity.dataitem.ItemCategory;
 import com.x.base.core.entity.tools.JpaObjectTools;
 import com.x.base.core.project.config.Config;
+import com.x.base.core.project.config.DumpRestoreData;
 import com.x.base.core.project.config.StorageMapping;
 import com.x.base.core.project.config.StorageMappings;
 import com.x.base.core.project.gson.XGsonBuilder;
@@ -46,7 +49,7 @@ import com.x.base.core.project.tools.ListTools;
 
 public class DumpData {
 
-	private static Logger logger = LoggerFactory.getLogger(DumpData.class);
+	private static final Logger LOGGER = LoggerFactory.getLogger(DumpData.class);
 
 	public boolean execute(String path) throws IOException, URISyntaxException {
 		Path dir = null;
@@ -56,7 +59,7 @@ public class DumpData {
 		} else {
 			dir = Paths.get(path);
 			if (dir.startsWith(Paths.get(Config.base()))) {
-				logger.warn("path can not in base directory.");
+				LOGGER.warn("path can not in base directory.");
 				return false;
 			}
 		}
@@ -87,8 +90,8 @@ public class DumpData {
 
 		public void run() {
 			try {
-				List<String> classNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader));
-				logger.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
+				List<String> classNames = this.entities(classLoader);
+				LOGGER.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
 				Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(),
 						DateTools.compact(start) + "_dump.xml");
 				PersistenceXmlHelper.write(xml.toString(), classNames, true, classLoader);
@@ -106,18 +109,16 @@ public class DumpData {
 					try {
 						@SuppressWarnings("unchecked")
 						Class<JpaObject> cls = (Class<JpaObject>) classLoader.loadClass(className);
-//						emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(),
-//								PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
 						emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(),
 								PersistenceXmlHelper.properties(cls.getName(), false));
 						em = emf.createEntityManager();
 						long estimateCount = estimateCount(em, cls);
-						logger.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(),
+						LOGGER.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(),
 								cls.getName(), estimateCount);
-						dump(cls, em, storageMappings, estimateCount);
+						dump(cls, em, storageMappings, Config.dumpRestoreData().getAttachStorage(), estimateCount);
 					} catch (Exception e) {
 						e.printStackTrace();
-						logger.error(new Exception(String.format("dump:%s error.", className), e));
+						LOGGER.error(new Exception(String.format("dump:%s error.", className), e));
 					} finally {
 						Thread.currentThread().setName(nameOfThread);
 						if (null != em) {
@@ -130,7 +131,7 @@ public class DumpData {
 				});
 				Files.write(dir.resolve("catalog.json"),
 						pureGsonDateFormated.toJson(catalog).getBytes(StandardCharsets.UTF_8));
-				logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(),
+				LOGGER.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(),
 
 						count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60);
 			} catch (Exception e) {
@@ -138,6 +139,24 @@ public class DumpData {
 			}
 		}
 
+		private List<String> entities(ClassLoader classLoader) throws Exception {
+			List<String> classNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader));
+			classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(),
+					Config.dumpRestoreData().getExcludes());
+			if (StringUtils.equalsIgnoreCase(DumpRestoreData.MODE_LITE, Config.dumpRestoreData().getMode())) {
+				return classNames.stream().filter(o -> {
+					try {
+						ContainerEntity containerEntity = classLoader.loadClass(o).getAnnotation(ContainerEntity.class);
+						return Objects.equals(containerEntity.reference(), ContainerEntity.Reference.strong);
+					} catch (Exception e) {
+						LOGGER.error(e);
+					}
+					return false;
+				}).collect(Collectors.toList());
+			}
+			return classNames;
+		}
+
 		private <T extends JpaObject> long estimateCount(EntityManager em, Class<T> cls) {
 			CriteriaBuilder cb = em.getCriteriaBuilder();
 			CriteriaQuery<Long> cq = cb.createQuery(Long.class);
@@ -167,8 +186,8 @@ public class DumpData {
 			return em.createQuery(cq).setMaxResults(size).getResultList();
 		}
 
-		private <T> void dump(Class<T> cls, EntityManager em, StorageMappings storageMappings, long total)
-				throws Exception {
+		private <T> void dump(Class<T> cls, EntityManager em, StorageMappings storageMappings, boolean attachStorage,
+				long total) throws Exception {
 			// 创建最终存储文件的目录
 			Path directory = dir.resolve(cls.getName());
 			Files.createDirectories(directory);
@@ -184,14 +203,14 @@ public class DumpData {
 				if (ListTools.isNotEmpty(list)) {
 					count = count + list.size();
 					id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME);
-					if (StorageObject.class.isAssignableFrom(cls)) {
+					if (StorageObject.class.isAssignableFrom(cls) && attachStorage) {
 						Path sub = directory.resolve(count + "");
 						Files.createDirectories(sub);
 						binary(list, sub, storageMappings);
 					}
 					Files.write(directory.resolve(count + ".json"),
 							pureGsonDateFormated.toJson(list).getBytes(StandardCharsets.UTF_8));
-					logger.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName());
+					LOGGER.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName());
 				}
 				em.clear();
 			} while (ListTools.isNotEmpty(list));

o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java

@@ -13,6 +13,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import javax.persistence.EntityManager;
@@ -23,6 +24,7 @@ import javax.persistence.criteria.CriteriaQuery;
 import javax.persistence.criteria.Predicate;
 import javax.persistence.criteria.Root;
 
+import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -56,26 +58,29 @@ public class RestoreData {
 
 	public boolean execute(String path) throws IOException, URISyntaxException {
 		Date start = new Date();
-		Path dir;
 		if (StringUtils.isEmpty(path)) {
 			LOGGER.warn("{}.", () -> "path is empty.");
 		}
 		ClassLoader classLoader = EntityClassLoaderTools.concreteClassLoader();
+		Path dir = dir(path);
+		Thread thread = new Thread(new RunnableImpl(dir, start, classLoader));
+		thread.start();
+		return true;
+	}
+
+	private Path dir(String path) throws IOException, URISyntaxException {
+		Path dir;
 		if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) {
 			dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + path);
 		} else {
 			dir = Paths.get(path);
 			if ((!Files.exists(dir)) || (!Files.isDirectory(dir))) {
-				LOGGER.warn("directory not exist: {}.", path);
-				return false;
+				throw new IllegalArgumentException("directory not exist: " + path + ".");
 			} else if (dir.startsWith(Paths.get(Config.base()))) {
-				LOGGER.warn("path can not in base directory.");
-				return false;
+				throw new IllegalArgumentException("path can not in base directory.");
 			}
 		}
-		Thread thread = new Thread(new RunnableImpl(dir, start, classLoader));
-		thread.start();
-		return true;
+		return dir;
 	}
 
 	public class RunnableImpl implements Runnable {
@@ -101,7 +106,7 @@ public class RestoreData {
 		@Override
 		public void run() {
 			try {
-				List<String> classNames = entities(catalog);
+				List<String> classNames = entities(catalog, classLoader);
 				LOGGER.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.toString());
 				Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(),
 						DateTools.compact(start) + "_restore.xml");
@@ -118,7 +123,7 @@ public class RestoreData {
 						Class<JpaObject> cls = (Class<JpaObject>) Thread.currentThread().getContextClassLoader()
 								.loadClass(className);
 						LOGGER.print("restore data({}/{}): {}.", idx.getAndAdd(1), classNames.size(), cls.getName());
-						long size = restore(cls, xml);
+						long size = restore(cls, Config.dumpRestoreData().getAttachStorage(), xml);
 						total.getAndAdd(size);
 					} catch (Exception e) {
 						LOGGER.error(new Exception(String.format("restore:%s error.", className), e));
@@ -132,10 +137,7 @@ public class RestoreData {
 			}
 		}
 
-		private long restore(Class<?> cls, Path xml) throws Exception {
-//			EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
-//					xml.getFileName().toString(),
-//					PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
+		private long restore(Class<?> cls, boolean attachStorage, Path xml) throws Exception {
 			EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
 					xml.getFileName().toString(), PersistenceXmlHelper.properties(cls.getName(), false));
 			EntityManager em = emf.createEntityManager();
@@ -157,24 +159,7 @@ public class RestoreData {
 				paths.stream().forEach(o -> {
 					LOGGER.print("restore {}/{} part of data:{}.", batch.getAndAdd(1), paths.size(), cls.getName());
 					try {
-						em.getTransaction().begin();
-						JsonArray raws = this.convert(o);
-						for (JsonElement json : raws) {
-							Object t = gson.fromJson(json, cls);
-							if (Objects.equals(Config.dumpRestoreData().getRestoreOverride(),
-									DumpRestoreData.RESTOREOVERRIDE_SKIPEXISTED)
-									&& (null != em.find(cls, BeanUtils.getBeanProperty(t, JpaObject.id_FIELDNAME)))) {
-								continue;
-							}
-							if (StorageObject.class.isAssignableFrom(cls)) {
-								Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString()));
-								this.binary(t, cls, sub, storageMappings);
-							}
-							em.persist(t);
-							count.getAndAdd(1);
-						}
-						em.getTransaction().commit();
-						em.clear();
+						restore(cls, o, em, attachStorage, storageMappings, count);
 					} catch (Exception e) {
 						LOGGER.error(new Exception(String.format("restore error with file:%s.", o.toString()), e));
 					}
@@ -189,6 +174,28 @@ public class RestoreData {
 			return count.longValue();
 		}
 
+		private void restore(Class<?> cls, Path o, EntityManager em, boolean attachStorage,
+				StorageMappings storageMappings, AtomicLong count) throws Exception {
+			em.getTransaction().begin();
+			JsonArray raws = this.convert(o);
+			for (JsonElement json : raws) {
+				Object t = gson.fromJson(json, cls);
+				if (Objects.equals(Config.dumpRestoreData().getRestoreOverride(),
+						DumpRestoreData.RESTOREOVERRIDE_SKIPEXISTED)
+						&& (null != em.find(cls, BeanUtils.getBeanProperty(t, JpaObject.id_FIELDNAME)))) {
+					continue;
+				}
+				if (StorageObject.class.isAssignableFrom(cls) && attachStorage) {
+					Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString()));
+					this.binary(t, cls, sub, storageMappings);
+				}
+				em.persist(t);
+				count.getAndAdd(1);
+			}
+			em.getTransaction().commit();
+			em.clear();
+		}
+
 		private List<Path> list(Path directory) throws IOException {
 			List<Path> list = new ArrayList<>();
 			try (Stream<Path> stream = Files.list(directory)) {
@@ -202,21 +209,30 @@ public class RestoreData {
 			return list;
 		}
 
-		private List<String> entities(DumpRestoreDataCatalog catalog) throws Exception {
-			List<String> containerEntityNames = new ArrayList<>();
-			containerEntityNames.addAll(JpaObjectTools.scanContainerEntityNames(classLoader));
-			List<String> classNames = new ArrayList<>();
-			classNames.addAll(catalog.keySet());
+		private List<String> entities(DumpRestoreDataCatalog catalog, ClassLoader classLoader) throws Exception {
+			List<String> containerEntityNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader));
+			if (StringUtils.equalsIgnoreCase(DumpRestoreData.MODE_LITE, Config.dumpRestoreData().getMode())) {
+				containerEntityNames = containerEntityNames.stream().filter(o -> {
+					try {
+						ContainerEntity containerEntity = classLoader.loadClass(o).getAnnotation(ContainerEntity.class);
+						return Objects.equals(containerEntity.reference(), ContainerEntity.Reference.strong);
+					} catch (Exception e) {
+						LOGGER.error(e);
+					}
+					return false;
+				}).collect(Collectors.toList());
+			}
+			List<String> classNames = new ArrayList<>(catalog.keySet());
 			classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(),
 					Config.dumpRestoreData().getExcludes());
-			return ListTools.includesExcludesWildcard(containerEntityNames, classNames, null);
+			return ListUtils.intersection(containerEntityNames, classNames);
 		}
 
 		@SuppressWarnings("unchecked")
 		private void binary(Object o, Class<?> cls, Path sub, StorageMappings storageMappings) throws Exception {
 			StorageObject so = (StorageObject) o;
 			StorageMapping mapping = null;
-			if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistribute())) {
+			if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistributeStorage())) {
 				mapping = storageMappings.random((Class<StorageObject>) cls);
 			} else {
 				mapping = storageMappings.get((Class<StorageObject>) cls, so.getStorage());