From 31897403870adc28cd3b6954208a1882a00c9613 Mon Sep 17 00:00:00 2001 From: zhourui Date: Wed, 8 Jun 2022 14:20:22 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0dumpData,restoreData=20?= =?UTF-8?q?=E4=B8=ADattachStorage=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/project/config/DumpRestoreData.java | 66 ++++++------- .../com/x/server/console/action/DumpData.java | 47 +++++++--- .../x/server/console/action/RestoreData.java | 94 +++++++++++-------- 3 files changed, 118 insertions(+), 89 deletions(-) diff --git a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java index 7f610ae200..7252bcce11 100644 --- a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java +++ b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java @@ -17,35 +17,31 @@ public class DumpRestoreData extends ConfigObject { return new DumpRestoreData(); } - public static final String TYPE_FULL = "full"; - public static final String TYPE_LITE = "lite"; + public static final String MODE_FULL = "full"; + public static final String MODE_LITE = "lite"; public static final String RESTOREOVERRIDE_CLEAN = "clean"; public static final String RESTOREOVERRIDE_SKIPEXISTED = "skipExisted"; - public static final String DEFAULT_TYPE = TYPE_LITE; public static final Boolean DEFAULT_PARALLEL = true; public static final Boolean DEFAULT_REDISTRIBUTE = true; public static final Boolean DEFAULT_EXCEPTIONINVALIDSTORAGE = true; + public static final Boolean DEFAULT_ATTACHSTORAGE = true; public static final String DEFAULT_ITEMCATEGORY = ""; public DumpRestoreData() { - this.enable = false; this.includes = new ArrayList<>(); this.excludes = new ArrayList<>(); - this.mode = DEFAULT_TYPE; + this.mode = MODE_LITE; this.parallel = DEFAULT_PARALLEL; - this.redistribute = DEFAULT_REDISTRIBUTE; + this.attachStorage = DEFAULT_ATTACHSTORAGE; this.exceptionInvalidStorage = DEFAULT_EXCEPTIONINVALIDSTORAGE; this.itemCategory = DEFAULT_ITEMCATEGORY; } - @FieldDescribe("是否启用.") - private Boolean enable; - - @FieldDescribe("导出导入包含对象,可以使用通配符*.") + @FieldDescribe("导出导入包含对象,可以使用通配符*,如仅导出待办数据:com.x.processplatform.core.entity.content.Task.") private List includes; - @FieldDescribe("导出导入排除对象,可以使用通配符*.") + @FieldDescribe("导出导入排除对象,可以使用通配符*,如不导出流程实例数据com.x.processplatform.core.entity.content.*") private List excludes; @FieldDescribe("导出数据模式,lite|full,默认使用lite") @@ -54,31 +50,41 @@ public class DumpRestoreData extends ConfigObject { @FieldDescribe("使用并行导出,默认true") private Boolean parallel; - @FieldDescribe("是否进行重新分布.") - private Boolean redistribute; - @FieldDescribe("无法获取storage是否升起错误.") private Boolean exceptionInvalidStorage; - @FieldDescribe("无法获取storage是否升起错误.") - private Boolean storageXXXXXXXXXXXXXXXX; - + @FieldDescribe("是否同时导入导出storage中存储的文件.") + private Boolean attachStorage; + + @FieldDescribe("是否进行重新分布.") + private Boolean redistributeStorage; + + public Boolean getRedistributeStorage() { + return redistributeStorage; + } + + public void setRedistributeStorage(Boolean redistributeStorage) { + this.redistributeStorage = redistributeStorage; + } + + public Boolean getAttachStorage() { + return attachStorage; + } + + public void setAttachStorage(Boolean attachStorage) { + this.attachStorage = attachStorage; + } + @FieldDescribe("数据导入方式,clean:清空重新导入,skipExisted:如果有相同id的数据跳过.默认方式为clean.") private String restoreOverride; @FieldDescribe("对于com.x.query.core.entity.Item的itemCategory进行单独过滤,可选值pp, cms, bbs, pp_dict.") private String itemCategory; - - public String getItemCategory() { return this.itemCategory; } - public Boolean getRedistribute() { - return BooleanUtils.isNotFalse(redistribute); - } - public Boolean getExceptionInvalidStorage() { return BooleanUtils.isNotFalse(exceptionInvalidStorage); } @@ -88,11 +94,7 @@ public class DumpRestoreData extends ConfigObject { } public String getMode() { - return StringUtils.equals(TYPE_FULL, mode) ? TYPE_FULL : TYPE_LITE; - } - - public Boolean getEnable() { - return BooleanUtils.isTrue(this.enable); + return StringUtils.equals(MODE_FULL, mode) ? MODE_FULL : MODE_LITE; } public List getIncludes() { @@ -123,18 +125,10 @@ public class DumpRestoreData extends ConfigObject { this.parallel = parallel; } - public void setRedistribute(Boolean redistribute) { - this.redistribute = redistribute; - } - public void setExceptionInvalidStorage(Boolean exceptionInvalidStorage) { this.exceptionInvalidStorage = exceptionInvalidStorage; } - public void setEnable(Boolean enable) { - this.enable = enable; - } - public void setMode(String mode) { this.mode = mode; } diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java b/o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java index 92240d501b..e76ee679f9 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java +++ b/o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java @@ -10,7 +10,9 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.persistence.EntityManager; @@ -36,6 +38,7 @@ import com.x.base.core.entity.dataitem.DataItem; import com.x.base.core.entity.dataitem.ItemCategory; import com.x.base.core.entity.tools.JpaObjectTools; import com.x.base.core.project.config.Config; +import com.x.base.core.project.config.DumpRestoreData; import com.x.base.core.project.config.StorageMapping; import com.x.base.core.project.config.StorageMappings; import com.x.base.core.project.gson.XGsonBuilder; @@ -46,7 +49,7 @@ import com.x.base.core.project.tools.ListTools; public class DumpData { - private static Logger logger = LoggerFactory.getLogger(DumpData.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DumpData.class); public boolean execute(String path) throws IOException, URISyntaxException { Path dir = null; @@ -56,7 +59,7 @@ public class DumpData { } else { dir = Paths.get(path); if (dir.startsWith(Paths.get(Config.base()))) { - logger.warn("path can not in base directory."); + LOGGER.warn("path can not in base directory."); return false; } } @@ -87,8 +90,8 @@ public class DumpData { public void run() { try { - List classNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader)); - logger.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start)); + List classNames = this.entities(classLoader); + LOGGER.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start)); Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(), DateTools.compact(start) + "_dump.xml"); PersistenceXmlHelper.write(xml.toString(), classNames, true, classLoader); @@ -106,18 +109,16 @@ public class DumpData { try { @SuppressWarnings("unchecked") Class cls = (Class) classLoader.loadClass(className); -// emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(), -// PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable())); emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(), PersistenceXmlHelper.properties(cls.getName(), false)); em = emf.createEntityManager(); long estimateCount = estimateCount(em, cls); - logger.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(), + LOGGER.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(), cls.getName(), estimateCount); - dump(cls, em, storageMappings, estimateCount); + dump(cls, em, storageMappings, Config.dumpRestoreData().getAttachStorage(), estimateCount); } catch (Exception e) { e.printStackTrace(); - logger.error(new Exception(String.format("dump:%s error.", className), e)); + LOGGER.error(new Exception(String.format("dump:%s error.", className), e)); } finally { Thread.currentThread().setName(nameOfThread); if (null != em) { @@ -130,7 +131,7 @@ public class DumpData { }); Files.write(dir.resolve("catalog.json"), pureGsonDateFormated.toJson(catalog).getBytes(StandardCharsets.UTF_8)); - logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(), + LOGGER.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(), count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60); } catch (Exception e) { @@ -138,6 +139,24 @@ public class DumpData { } } + private List entities(ClassLoader classLoader) throws Exception { + List classNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader)); + classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(), + Config.dumpRestoreData().getExcludes()); + if (StringUtils.equalsIgnoreCase(DumpRestoreData.MODE_LITE, Config.dumpRestoreData().getMode())) { + return classNames.stream().filter(o -> { + try { + ContainerEntity containerEntity = classLoader.loadClass(o).getAnnotation(ContainerEntity.class); + return Objects.equals(containerEntity.reference(), ContainerEntity.Reference.strong); + } catch (Exception e) { + LOGGER.error(e); + } + return false; + }).collect(Collectors.toList()); + } + return classNames; + } + private long estimateCount(EntityManager em, Class cls) { CriteriaBuilder cb = em.getCriteriaBuilder(); CriteriaQuery cq = cb.createQuery(Long.class); @@ -167,8 +186,8 @@ public class DumpData { return em.createQuery(cq).setMaxResults(size).getResultList(); } - private void dump(Class cls, EntityManager em, StorageMappings storageMappings, long total) - throws Exception { + private void dump(Class cls, EntityManager em, StorageMappings storageMappings, boolean attachStorage, + long total) throws Exception { // 创建最终存储文件的目录 Path directory = dir.resolve(cls.getName()); Files.createDirectories(directory); @@ -184,14 +203,14 @@ public class DumpData { if (ListTools.isNotEmpty(list)) { count = count + list.size(); id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME); - if (StorageObject.class.isAssignableFrom(cls)) { + if (StorageObject.class.isAssignableFrom(cls) && attachStorage) { Path sub = directory.resolve(count + ""); Files.createDirectories(sub); binary(list, sub, storageMappings); } Files.write(directory.resolve(count + ".json"), pureGsonDateFormated.toJson(list).getBytes(StandardCharsets.UTF_8)); - logger.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName()); + LOGGER.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName()); } em.clear(); } while (ListTools.isNotEmpty(list)); diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java b/o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java index 1f18b8216a..423cf56193 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java +++ b/o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.persistence.EntityManager; @@ -23,6 +24,7 @@ import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; +import org.apache.commons.collections4.ListUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; @@ -56,26 +58,29 @@ public class RestoreData { public boolean execute(String path) throws IOException, URISyntaxException { Date start = new Date(); - Path dir; if (StringUtils.isEmpty(path)) { LOGGER.warn("{}.", () -> "path is empty."); } ClassLoader classLoader = EntityClassLoaderTools.concreteClassLoader(); + Path dir = dir(path); + Thread thread = new Thread(new RunnableImpl(dir, start, classLoader)); + thread.start(); + return true; + } + + private Path dir(String path) throws IOException, URISyntaxException { + Path dir; if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) { dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + path); } else { dir = Paths.get(path); if ((!Files.exists(dir)) || (!Files.isDirectory(dir))) { - LOGGER.warn("directory not exist: {}.", path); - return false; + throw new IllegalArgumentException("directory not exist: " + path + "."); } else if (dir.startsWith(Paths.get(Config.base()))) { - LOGGER.warn("path can not in base directory."); - return false; + throw new IllegalArgumentException("path can not in base directory."); } } - Thread thread = new Thread(new RunnableImpl(dir, start, classLoader)); - thread.start(); - return true; + return dir; } public class RunnableImpl implements Runnable { @@ -101,7 +106,7 @@ public class RestoreData { @Override public void run() { try { - List classNames = entities(catalog); + List classNames = entities(catalog, classLoader); LOGGER.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.toString()); Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(), DateTools.compact(start) + "_restore.xml"); @@ -118,7 +123,7 @@ public class RestoreData { Class cls = (Class) Thread.currentThread().getContextClassLoader() .loadClass(className); LOGGER.print("restore data({}/{}): {}.", idx.getAndAdd(1), classNames.size(), cls.getName()); - long size = restore(cls, xml); + long size = restore(cls, Config.dumpRestoreData().getAttachStorage(), xml); total.getAndAdd(size); } catch (Exception e) { LOGGER.error(new Exception(String.format("restore:%s error.", className), e)); @@ -132,10 +137,7 @@ public class RestoreData { } } - private long restore(Class cls, Path xml) throws Exception { -// EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), -// xml.getFileName().toString(), -// PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable())); + private long restore(Class cls, boolean attachStorage, Path xml) throws Exception { EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(), PersistenceXmlHelper.properties(cls.getName(), false)); EntityManager em = emf.createEntityManager(); @@ -157,24 +159,7 @@ public class RestoreData { paths.stream().forEach(o -> { LOGGER.print("restore {}/{} part of data:{}.", batch.getAndAdd(1), paths.size(), cls.getName()); try { - em.getTransaction().begin(); - JsonArray raws = this.convert(o); - for (JsonElement json : raws) { - Object t = gson.fromJson(json, cls); - if (Objects.equals(Config.dumpRestoreData().getRestoreOverride(), - DumpRestoreData.RESTOREOVERRIDE_SKIPEXISTED) - && (null != em.find(cls, BeanUtils.getBeanProperty(t, JpaObject.id_FIELDNAME)))) { - continue; - } - if (StorageObject.class.isAssignableFrom(cls)) { - Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString())); - this.binary(t, cls, sub, storageMappings); - } - em.persist(t); - count.getAndAdd(1); - } - em.getTransaction().commit(); - em.clear(); + restore(cls, o, em, attachStorage, storageMappings, count); } catch (Exception e) { LOGGER.error(new Exception(String.format("restore error with file:%s.", o.toString()), e)); } @@ -189,6 +174,28 @@ public class RestoreData { return count.longValue(); } + private void restore(Class cls, Path o, EntityManager em, boolean attachStorage, + StorageMappings storageMappings, AtomicLong count) throws Exception { + em.getTransaction().begin(); + JsonArray raws = this.convert(o); + for (JsonElement json : raws) { + Object t = gson.fromJson(json, cls); + if (Objects.equals(Config.dumpRestoreData().getRestoreOverride(), + DumpRestoreData.RESTOREOVERRIDE_SKIPEXISTED) + && (null != em.find(cls, BeanUtils.getBeanProperty(t, JpaObject.id_FIELDNAME)))) { + continue; + } + if (StorageObject.class.isAssignableFrom(cls) && attachStorage) { + Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString())); + this.binary(t, cls, sub, storageMappings); + } + em.persist(t); + count.getAndAdd(1); + } + em.getTransaction().commit(); + em.clear(); + } + private List list(Path directory) throws IOException { List list = new ArrayList<>(); try (Stream stream = Files.list(directory)) { @@ -202,21 +209,30 @@ public class RestoreData { return list; } - private List entities(DumpRestoreDataCatalog catalog) throws Exception { - List containerEntityNames = new ArrayList<>(); - containerEntityNames.addAll(JpaObjectTools.scanContainerEntityNames(classLoader)); - List classNames = new ArrayList<>(); - classNames.addAll(catalog.keySet()); + private List entities(DumpRestoreDataCatalog catalog, ClassLoader classLoader) throws Exception { + List containerEntityNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader)); + if (StringUtils.equalsIgnoreCase(DumpRestoreData.MODE_LITE, Config.dumpRestoreData().getMode())) { + containerEntityNames = containerEntityNames.stream().filter(o -> { + try { + ContainerEntity containerEntity = classLoader.loadClass(o).getAnnotation(ContainerEntity.class); + return Objects.equals(containerEntity.reference(), ContainerEntity.Reference.strong); + } catch (Exception e) { + LOGGER.error(e); + } + return false; + }).collect(Collectors.toList()); + } + List classNames = new ArrayList<>(catalog.keySet()); classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(), Config.dumpRestoreData().getExcludes()); - return ListTools.includesExcludesWildcard(containerEntityNames, classNames, null); + return ListUtils.intersection(containerEntityNames, classNames); } @SuppressWarnings("unchecked") private void binary(Object o, Class cls, Path sub, StorageMappings storageMappings) throws Exception { StorageObject so = (StorageObject) o; StorageMapping mapping = null; - if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistribute())) { + if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistributeStorage())) { mapping = storageMappings.random((Class) cls); } else { mapping = storageMappings.get((Class) cls, so.getStorage()); -- GitLab