diff --git a/o2server/configSample/dumpRestoreData.json b/o2server/configSample/dumpRestoreData.json index 0805916115033fdd5440d26c3901a8909320b96f..9cd3b94a3b704328524eafb898a4e02346dac77d 100644 --- a/o2server/configSample/dumpRestoreData.json +++ b/o2server/configSample/dumpRestoreData.json @@ -2,11 +2,15 @@ "enable": false, "includes": [], "excludes": [], - "batchSize": 1000.0, "mode": "lite", + "parallel": true, + "redistribute": true, + "exceptionInvalidStorage": true, "###enable": "是否启用.###", "###includes": "导出导入包含对象,可以使用通配符*.###", "###excludes": "导出导入排除对象,可以使用通配符*.###", - "###batchSize": "批量对象数量.###", - "###mode": "导出数据模式,lite|full,默认使用lite###" + "###mode": "导出数据模式,lite|full,默认使用lite###", + "###parallel": "使用并行导出,默认true###", + "###redistribute": "是否进行重新分布.###", + "###exceptionInvalidStorage": "无法获取storage是否升起错误.###" } \ No newline at end of file diff --git a/o2server/configSample/node_127.0.0.1.json b/o2server/configSample/node_127.0.0.1.json index 064bffa7d320f660ceacd0ae15c8b10cfd99a6f2..2c85fd6f484f94225249b3c40fe89a6c261d7fae 100644 --- a/o2server/configSample/node_127.0.0.1.json +++ b/o2server/configSample/node_127.0.0.1.json @@ -138,16 +138,6 @@ "###size": "最大保留份数,超过将自动删除最久的数据.###", "###path": "备份路径###" }, - "dumpStorage": { - "enable": true, - "cron": "", - "size": 7.0, - "path": "", - "###enable": "是否启用,默认每天凌晨4点进行备份.###", - "###cron": "定时任务cron表达式###", - "###size": "最大保留份数,超过将自动删除最久的数据.###", - "###path": "备份路径###" - }, "restoreData": { "enable": false, "cron": "", @@ -156,14 +146,6 @@ "###cron": "定时任务cron表达式###", "###path": "恢复路径###" }, - "restoreStorage": { - "enable": false, - "cron": "", - "path": "", - "###enable": "是否启用.###", - "###cron": "定时任务cron表达式###", - "###path": "恢复路径###" - }, "nodeAgentEnable": true, "nodeAgentPort": 20010.0, "nodeAgentEncrypt": true, @@ -178,9 +160,7 @@ "###storage": "Storage服务器配置###", "###logLevel": "日志级别,默认当前节点的slf4j日志级别,通过系统变量\"org.slf4j.simpleLogger.defaultLogLevel\"设置到当前jvm中.###", "###dumpData": "定时数据导出配置###", - "###dumpStorage": "定时存储文件导出配置###", "###restoreData": "定时数据导入配置###", - "###restoreStorage": "定时存储文件导入配置###", "###logSize": "日志文件保留天数.###", "###auditLogSize": "审计日志文件保留天数.###", "###nodeAgentEnable": "是否启用节点代理###", diff --git a/o2server/pom.xml b/o2server/pom.xml index 58ee7c45bf163c36b486e4989cb0b63d866f5b21..61b7f5e23c5cec1ea4da9e49daab81240dc8a7db 100644 --- a/o2server/pom.xml +++ b/o2server/pom.xml @@ -494,7 +494,7 @@ commons-io commons-io - 2.6 + 2.7 org.apache.commons diff --git a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/build/CheckCore.java b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/build/CheckCore.java index a8d29903fc1335da2d48e198f067cd473f4e66da..a4e77c1c8c044864da7a865e03a6185117316e45 100644 --- a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/build/CheckCore.java +++ b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/build/CheckCore.java @@ -19,6 +19,8 @@ import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.openjpa.persistence.jdbc.ContainerTable; import com.x.base.core.entity.JpaObject; +import com.x.base.core.entity.JsonProperties; +import com.x.base.core.entity.StorageObject; import com.x.base.core.entity.annotation.ContainerEntity; import com.x.base.core.project.annotation.FieldDescribe; import com.x.base.core.project.annotation.Module; @@ -44,6 +46,7 @@ public class CheckCore { checkTableNameUniqueConstraintName(classes); checkIdCreateTimeUpdateTimeSequenceIndex(classes); checkEnum(classes); + checkDumpSize(classes); } } @@ -215,17 +218,33 @@ public class CheckCore { for (Class cls : classes) { List fields = FieldUtils.getFieldsListWithAnnotation(cls, Column.class); for (Field field : fields) { - if ((!String.class.isAssignableFrom(field.getType())) && (!field.getType().isEnum())) { - Column column = field.getAnnotation(Column.class); - if (column.length() != 255) { - System.err.println(String.format("checkColumnLength error: class: %s, field: %s.", - cls.getName(), field.getName())); + if ((!String.class.isAssignableFrom(field.getType())) && (!field.getType().isEnum()) + && (!JsonProperties.class.isAssignableFrom(field.getType()))) { + Lob lob = cls.getAnnotation(Lob.class); + if (null != lob) { + Column column = field.getAnnotation(Column.class); + if (column.length() > 255) { + System.err.println(String.format("checkColumnLength error: class: %s, field: %s.", + cls.getName(), field.getName())); + } } } } } } + /* 检查StorageObject的dumpSize是否设置正确 */ + public static void checkDumpSize(List> classes) throws Exception { + for (Class cls : classes) { + if (StorageObject.class.isAssignableFrom(cls)) { + ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class); + if (containerEntity.dumpSize() > 10) { + System.err.println(String.format("checkDumpSize error: class: %s.", cls.getName())); + } + } + } + } + public static void checkIdUnique(List> classes) throws Exception { for (Class cls : classes) { Field idField = FieldUtils.getField(cls, JpaObject.id_FIELDNAME, true); diff --git a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java index f7b1a3ae15636a97d4bcc89bd38407e4073d66a3..08b9b724b4514ffd6ec2fa5284c54e03890f4da8 100644 --- a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java +++ b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java @@ -11,22 +11,27 @@ import com.x.base.core.project.tools.ListTools; public class DumpRestoreData extends ConfigObject { - public static String TYPE_FULL = "full"; - public static String TYPE_LITE = "lite"; + private static final long serialVersionUID = 8910820385137391619L; public static DumpRestoreData defaultInstance() { return new DumpRestoreData(); } - public static final int default_batchSize = 1000; - public static final String default_type = TYPE_LITE; + public static final String TYPE_FULL = "full"; + public static final String TYPE_LITE = "lite"; + public static final String DEFAULT_TYPE = TYPE_LITE; + public static final Boolean DEFAULT_PARALLEL = true; + public static final Boolean DEFAULT_REDISTRIBUTE = true; + public static final Boolean DEFAULT_EXCEPTIONINVALIDSTORAGE = true; public DumpRestoreData() { this.enable = false; - this.includes = new ArrayList(); - this.excludes = new ArrayList(); - this.batchSize = default_batchSize; - this.mode = default_type; + this.includes = new ArrayList<>(); + this.excludes = new ArrayList<>(); + this.mode = DEFAULT_TYPE; + this.parallel = DEFAULT_PARALLEL; + this.redistribute = DEFAULT_REDISTRIBUTE; + this.exceptionInvalidStorage = DEFAULT_EXCEPTIONINVALIDSTORAGE; } @FieldDescribe("是否启用.") @@ -38,12 +43,30 @@ public class DumpRestoreData extends ConfigObject { @FieldDescribe("导出导入排除对象,可以使用通配符*.") private List excludes; - @FieldDescribe("批量对象数量.") - private Integer batchSize; - @FieldDescribe("导出数据模式,lite|full,默认使用lite") private String mode; + @FieldDescribe("使用并行导出,默认true") + private Boolean parallel; + + @FieldDescribe("是否进行重新分布.") + private Boolean redistribute; + + @FieldDescribe("无法获取storage是否升起错误.") + private Boolean exceptionInvalidStorage; + + public Boolean getRedistribute() { + return BooleanUtils.isNotFalse(redistribute); + } + + public Boolean getExceptionInvalidStorage() { + return BooleanUtils.isNotFalse(exceptionInvalidStorage); + } + + public Boolean getParallel() { + return BooleanUtils.isNotFalse(parallel); + } + public String getMode() { return StringUtils.equals(TYPE_FULL, mode) ? TYPE_FULL : TYPE_LITE; } @@ -68,13 +91,6 @@ public class DumpRestoreData extends ConfigObject { return list; } - public Integer getBatchSize() { - if ((null == this.batchSize) || (this.batchSize < 1)) { - return default_batchSize; - } - return this.batchSize; - } - public void setIncludes(List includes) { this.includes = includes; } @@ -83,8 +99,16 @@ public class DumpRestoreData extends ConfigObject { this.excludes = excludes; } - public void setBatchSize(Integer batchSize) { - this.batchSize = batchSize; + public void setParallel(Boolean parallel) { + this.parallel = parallel; + } + + public void setRedistribute(Boolean redistribute) { + this.redistribute = redistribute; + } + + public void setExceptionInvalidStorage(Boolean exceptionInvalidStorage) { + this.exceptionInvalidStorage = exceptionInvalidStorage; } public void setEnable(Boolean enable) { diff --git a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/Node.java b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/Node.java index 942115db037933775c3dee7d471e50afce5250b5..906af18518857fe0647bf588bdc4fa6b4b9b9743 100644 --- a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/Node.java +++ b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/Node.java @@ -1,16 +1,16 @@ package com.x.base.core.project.config; -import com.x.base.core.project.annotation.FieldDescribe; -import com.x.base.core.project.tools.DateTools; - import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; +import com.x.base.core.project.annotation.FieldDescribe; +import com.x.base.core.project.tools.DateTools; + public class Node extends ConfigObject { - public static final Integer default_nodeAgentPort = 20010; - public static final String default_banner = "O2OA"; - public static final Integer default_logSize = 14; + public static final Integer DEFAULT_NODEAGENTPORT = 20010; + public static final String DEFAULT_BANNER = "O2OA"; + public static final Integer DEFAULT_LOGSIZE = 14; public static Node defaultInstance() { Node o = new Node(); @@ -23,12 +23,10 @@ public class Node extends ConfigObject { o.storage = StorageServer.defaultInstance(); o.logLevel = "warn"; o.dumpData = new ScheduleDumpData(); - o.dumpStorage = new ScheduleDumpStorage(); o.restoreData = new ScheduleRestoreData(); - o.restoreStorage = new ScheduleRestoreStorage(); o.nodeAgentEnable = true; o.nodeAgentEncrypt = true; - o.nodeAgentPort = default_nodeAgentPort; + o.nodeAgentPort = DEFAULT_NODEAGENTPORT; o.quickStartWebApp = false; o.autoStart = true; return o; @@ -52,12 +50,8 @@ public class Node extends ConfigObject { private String logLevel; @FieldDescribe("定时数据导出配置") private ScheduleDumpData dumpData; - @FieldDescribe("定时存储文件导出配置") - private ScheduleDumpStorage dumpStorage; @FieldDescribe("定时数据导入配置") private ScheduleRestoreData restoreData; - @FieldDescribe("定时存储文件导入配置") - private ScheduleRestoreStorage restoreStorage; @FieldDescribe("日志文件保留天数.") private Integer logSize; @FieldDescribe("审计日志文件保留天数.") @@ -83,7 +77,6 @@ public class Node extends ConfigObject { } /* 20191009兼容centerServer end */ - public Boolean getEraseContentEnable() { return BooleanUtils.isNotFalse(eraseContentEnable); } @@ -136,14 +129,14 @@ public class Node extends ConfigObject { } public String getBanner() { - return StringUtils.isBlank(this.banner) ? default_banner : this.banner; + return StringUtils.isBlank(this.banner) ? DEFAULT_BANNER : this.banner; } public Integer logSize() { if ((this.logSize != null) && (this.logSize > 0)) { return this.logSize; } - return default_logSize; + return DEFAULT_LOGSIZE; } public Boolean getQuickStartWebApp() { @@ -152,7 +145,7 @@ public class Node extends ConfigObject { public Integer nodeAgentPort() { if (null == this.nodeAgentPort || this.nodeAgentPort < 0) { - return default_nodeAgentPort; + return DEFAULT_NODEAGENTPORT; } return this.nodeAgentPort; } @@ -173,18 +166,10 @@ public class Node extends ConfigObject { return (dumpData == null) ? new ScheduleDumpData() : this.dumpData; } - public ScheduleDumpStorage dumpStorage() { - return (dumpStorage == null) ? new ScheduleDumpStorage() : this.dumpStorage; - } - public ScheduleRestoreData restoreData() { return (restoreData == null) ? new ScheduleRestoreData() : this.restoreData; } - public ScheduleRestoreStorage restoreStorage() { - return (restoreStorage == null) ? new ScheduleRestoreStorage() : this.restoreStorage; - } - public static class ScheduleDumpData extends ConfigObject { public static ScheduleDumpData defaultInstance() { @@ -225,46 +210,6 @@ public class Node extends ConfigObject { } - public static class ScheduleDumpStorage extends ConfigObject { - - public static ScheduleDumpStorage defaultInstance() { - return new ScheduleDumpStorage(); - } - - public boolean available() { - return DateTools.cronAvailable(this.cron()); - } - - @FieldDescribe("是否启用,默认每天凌晨4点进行备份.") - private Boolean enable = true; - - @FieldDescribe("定时任务cron表达式") - private String cron = ""; - - @FieldDescribe("最大保留份数,超过将自动删除最久的数据.") - private Integer size = 7; - - @FieldDescribe("备份路径") - private String path = ""; - - public Boolean enable() { - return (BooleanUtils.isTrue(this.enable)) ? true : false; - } - - public String cron() { - return (null == cron) ? "5 0 4 * * ?" : this.cron; - } - - public Integer size() { - return (null == size) ? 14 : this.size; - } - - public String path() { - return StringUtils.trim(path); - } - - } - public static class ScheduleRestoreData extends ConfigObject { public static ScheduleRestoreData defaultInstance() { @@ -298,37 +243,4 @@ public class Node extends ConfigObject { } - public static class ScheduleRestoreStorage extends ConfigObject { - - public static ScheduleRestoreStorage defaultInstance() { - return new ScheduleRestoreStorage(); - } - - public boolean available() { - return DateTools.cronAvailable(this.cron) && StringUtils.isNotEmpty(this.path); - } - - @FieldDescribe("是否启用.") - private Boolean enable = false; - - @FieldDescribe("定时任务cron表达式") - private String cron = ""; - - @FieldDescribe("恢复路径") - private String path = ""; - - public Boolean enable() { - return (BooleanUtils.isTrue(this.enable)) ? true : false; - } - - public String cron() { - return (null == cron) ? "" : this.cron; - } - - public String path() { - return StringUtils.trim(path); - } - - } - } diff --git a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/tools/StringTools.java b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/tools/StringTools.java index c0d31ab0804a6cd00bd92a1ebb0e8aba50692adb..fb5d1afa1d0e7cf3130fadc27ff9e951527830a4 100644 --- a/o2server/x_base_core_project/src/main/java/com/x/base/core/project/tools/StringTools.java +++ b/o2server/x_base_core_project/src/main/java/com/x/base/core/project/tools/StringTools.java @@ -35,7 +35,26 @@ public class StringTools { /** 中文,英文,数字,-,.· 【】（） */ public static final Pattern SIMPLY_REGEX = Pattern .compile("^[\u4e00-\u9fa5a-zA-Z0-9\\_\\(\\)\\-\\ \\.\\ \\·\\【\\】\\（\\）]*$"); - public static final Pattern FILENAME_REGEX = Pattern.compile("[^/\\\\<>*?|\"]+(\\.?)[^/\\\\<>*?|\"]+"); + /** + * MSDN + * https://docs.microsoft.com/zh-cn/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#file_and_directory_names + */ + public static final Pattern FILENAME_REGEX = Pattern.compile( + "# Match a valid Windows filename (unspecified file system). \n" + + "^ # Anchor to start of string. \n" + + "(?! # Assert filename is not: CON, PRN, \n" + + " (?: # AUX, NUL, COM1, COM2, COM3, COM4, \n" + + " CON|PRN|AUX|NUL| # COM5, COM6, COM7, COM8, COM9, \n" + + " COM[1-9]|LPT[1-9] # LPT1, LPT2, LPT3, LPT4, LPT5, \n" + + " ) # LPT6, LPT7, LPT8, and LPT9... \n" + + " (?:\\.[^.]*)? # followed by optional extension \n" + + " $ # and end of string \n" + + ") # End negative lookahead assertion. \n" + + "[^<>:\"/\\\\|?*\\x00-\\x1F]* # Zero or more valid filename chars.\n" + + "[^<>:\"/\\\\|?*\\x00-\\x1F\\ .] # Last char is not a space or dot. \n" + + "$ # Anchor to end of string. ", + Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.COMMENTS); + /** * RFC822 compliant regex adapted for Java * http://stackoverflow.com/questions/8204680/java-regex-email @@ -480,37 +499,37 @@ public class StringTools { while (tok.hasMoreTokens()) { String nextTok = tok.nextToken(); switch (state) { - case inQuote: - if ("\'".equals(nextTok)) { - lastTokenHasBeenQuoted = true; - state = normal; - } else { - current.append(nextTok); - } - break; - case inDoubleQuote: - if ("\"".equals(nextTok)) { - lastTokenHasBeenQuoted = true; - state = normal; - } else { - current.append(nextTok); - } - break; - default: - if ("\'".equals(nextTok)) { - state = inQuote; - } else if ("\"".equals(nextTok)) { - state = inDoubleQuote; - } else if (" ".equals(nextTok)) { - if (lastTokenHasBeenQuoted || current.length() > 0) { - result.add(current.toString()); - current.setLength(0); - } - } else { - current.append(nextTok); + case inQuote: + if ("\'".equals(nextTok)) { + lastTokenHasBeenQuoted = true; + state = normal; + } else { + current.append(nextTok); + } + break; + case inDoubleQuote: + if ("\"".equals(nextTok)) { + lastTokenHasBeenQuoted = true; + state = normal; + } else { + current.append(nextTok); + } + break; + default: + if ("\'".equals(nextTok)) { + state = inQuote; + } else if ("\"".equals(nextTok)) { + state = inDoubleQuote; + } else if (" ".equals(nextTok)) { + if (lastTokenHasBeenQuoted || current.length() > 0) { + result.add(current.toString()); + current.setLength(0); } - lastTokenHasBeenQuoted = false; - break; + } else { + current.append(nextTok); + } + lastTokenHasBeenQuoted = false; + break; } } if (lastTokenHasBeenQuoted || current.length() > 0) { diff --git a/o2server/x_bbs_core_entity/src/main/java/com/x/bbs/entity/BBSSubjectAttachment.java b/o2server/x_bbs_core_entity/src/main/java/com/x/bbs/entity/BBSSubjectAttachment.java index d8cc92abfff41f8ff08d98fa01fd1f082456a48c..daeeae00155e1fb12810c968f8e13e5b5b3e8328 100644 --- a/o2server/x_bbs_core_entity/src/main/java/com/x/bbs/entity/BBSSubjectAttachment.java +++ b/o2server/x_bbs_core_entity/src/main/java/com/x/bbs/entity/BBSSubjectAttachment.java @@ -29,7 +29,7 @@ import com.x.base.core.project.tools.DateTools; /** * 附件信息管理表 */ -@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) +@ContainerEntity(dumpSize = 10, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) @Entity @Table(name = PersistenceProperties.BBSSubjectAttachment.table, uniqueConstraints = { @UniqueConstraint(name = PersistenceProperties.BBSSubjectAttachment.table + JpaObject.IndexNameMiddle diff --git a/o2server/x_cms_core_entity/src/main/java/com/x/cms/core/entity/FileInfo.java b/o2server/x_cms_core_entity/src/main/java/com/x/cms/core/entity/FileInfo.java index b2fda22e917cbe67b7699db721116f784ea9e37a..0440805f353e0c7c0c186656795d186eef077c53 100644 --- a/o2server/x_cms_core_entity/src/main/java/com/x/cms/core/entity/FileInfo.java +++ b/o2server/x_cms_core_entity/src/main/java/com/x/cms/core/entity/FileInfo.java @@ -39,7 +39,7 @@ import com.x.base.core.project.tools.DateTools; * 内容管理应用目录分类信息 * */ -@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) +@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) @Entity @Table(name = PersistenceProperties.FileInfo.table, uniqueConstraints = { @UniqueConstraint(name = PersistenceProperties.FileInfo.table + JpaObject.IndexNameMiddle diff --git a/o2server/x_console/src/main/java/com/x/server/console/DumpStorageTask.java b/o2server/x_console/src/main/java/com/x/server/console/DumpStorageTask.java index 6b345653ea27375ee6880813f1a910d0fda7d269..7fd87410256ff7f9d3888078d056822df0eff974 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/DumpStorageTask.java +++ b/o2server/x_console/src/main/java/com/x/server/console/DumpStorageTask.java @@ -1,56 +1,56 @@ -package com.x.server.console; - -import java.io.File; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.FalseFileFilter; -import org.apache.commons.io.filefilter.RegexFileFilter; -import org.quartz.Job; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -import com.x.base.core.project.config.Config; -import com.x.base.core.project.logger.Logger; -import com.x.base.core.project.logger.LoggerFactory; -import com.x.server.console.action.DumpStorage; - -public class DumpStorageTask implements Job { - - private static Logger logger = LoggerFactory.getLogger(DumpStorageTask.class); - - @Override - public void execute(JobExecutionContext arg0) throws JobExecutionException { - try { - logger.print("schedule dump storage task start."); - DumpStorage action = new DumpStorage(); - action.execute(Config.currentNode().dumpStorage().path()); - if (Config.currentNode().dumpStorage().size() > 0) { - File dir = new File(Config.base(), "local/dump"); - List list = new ArrayList<>(); - if (dir.exists() && dir.isDirectory()) { - for (File f : FileUtils.listFilesAndDirs(dir, FalseFileFilter.FALSE, new RegexFileFilter( - "^dumpStorage_[1,2][0,9][0-9][0-9][0,1][0-9][0-3][0-9][0-5][0-9][0-5][0-9][0-5][0-9]$"))) { - if (dir != f) { - list.add(f); - } - } - list = list.stream().sorted(Comparator.comparing(File::getName).reversed()) - .collect(Collectors.toList()); - if (list.size() > Config.currentNode().dumpStorage().size()) { - for (int i = Config.currentNode().dumpStorage().size(); i < list.size(); i++) { - File file = list.get(i); - logger.print("dumpStorageTask delete{}.", file.getAbsolutePath()); - FileUtils.forceDelete(file); - } - } - } - } - } catch (Exception e) { - e.printStackTrace(); - } - } -} \ No newline at end of file +//package com.x.server.console; +// +//import java.io.File; +//import java.util.ArrayList; +//import java.util.Comparator; +//import java.util.List; +//import java.util.stream.Collectors; +// +//import org.apache.commons.io.FileUtils; +//import org.apache.commons.io.filefilter.FalseFileFilter; +//import org.apache.commons.io.filefilter.RegexFileFilter; +//import org.quartz.Job; +//import org.quartz.JobExecutionContext; +//import org.quartz.JobExecutionException; +// +//import com.x.base.core.project.config.Config; +//import com.x.base.core.project.logger.Logger; +//import com.x.base.core.project.logger.LoggerFactory; +//import com.x.server.console.action.DumpStorage; +// +//public class DumpStorageTask implements Job { +// +// private static Logger logger = LoggerFactory.getLogger(DumpStorageTask.class); +// +// @Override +// public void execute(JobExecutionContext arg0) throws JobExecutionException { +// try { +// logger.print("schedule dump storage task start."); +// DumpStorage action = new DumpStorage(); +// action.execute(Config.currentNode().dumpStorage().path()); +// if (Config.currentNode().dumpStorage().size() > 0) { +// File dir = new File(Config.base(), "local/dump"); +// List list = new ArrayList<>(); +// if (dir.exists() && dir.isDirectory()) { +// for (File f : FileUtils.listFilesAndDirs(dir, FalseFileFilter.FALSE, new RegexFileFilter( +// "^dumpStorage_[1,2][0,9][0-9][0-9][0,1][0-9][0-3][0-9][0-5][0-9][0-5][0-9][0-5][0-9]$"))) { +// if (dir != f) { +// list.add(f); +// } +// } +// list = list.stream().sorted(Comparator.comparing(File::getName).reversed()) +// .collect(Collectors.toList()); +// if (list.size() > Config.currentNode().dumpStorage().size()) { +// for (int i = Config.currentNode().dumpStorage().size(); i < list.size(); i++) { +// File file = list.get(i); +// logger.print("dumpStorageTask delete{}.", file.getAbsolutePath()); +// FileUtils.forceDelete(file); +// } +// } +// } +// } +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } +//} \ No newline at end of file diff --git a/o2server/x_console/src/main/java/com/x/server/console/Main.java b/o2server/x_console/src/main/java/com/x/server/console/Main.java index bc2973fc3072b91c3093324b5e960872057770de..622784a11ca38cf09099365da0c7297efe5e42a8 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/Main.java +++ b/o2server/x_console/src/main/java/com/x/server/console/Main.java @@ -3,8 +3,11 @@ package com.x.server.console; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; +import java.io.IOException; import java.io.InputStreamReader; import java.io.RandomAccessFile; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; @@ -12,12 +15,22 @@ import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.channels.FileChannel.MapMode; import java.nio.channels.FileLock; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; import java.util.regex.Matcher; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.quartz.Scheduler; + import com.x.base.core.project.config.ApplicationServer; import com.x.base.core.project.config.CenterServer; import com.x.base.core.project.config.Config; @@ -34,14 +47,6 @@ import com.x.server.console.action.ActionVersion; import com.x.server.console.log.LogTools; import com.x.server.console.server.Servers; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.jetty.deploy.App; -import org.eclipse.jetty.deploy.DeploymentManager; -import org.quartz.Scheduler; - public class Main { private static final String MANIFEST_FILENAME = "manifest.cfg"; @@ -50,6 +55,7 @@ public class Main { public static void main(String[] args) throws Exception { String base = getBasePath(); + pid(base); scanWar(base); loadJars(base); /* getVersion需要FileUtils在后面运行 */ @@ -620,4 +626,12 @@ public class Main { return false; } + private static void pid(String base) throws IOException { + RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); + String jvmName = runtimeBean.getName(); + long pid = Long.parseLong(jvmName.split("@")[0]); + Path path = Paths.get(base, "pid.log"); + Files.write(path, Long.toString(pid).getBytes(), StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + } } \ No newline at end of file diff --git a/o2server/x_console/src/main/java/com/x/server/console/RestoreStorageTask.java b/o2server/x_console/src/main/java/com/x/server/console/RestoreStorageTask.java index c6314daa189d5ab8a2b26b552650ee5e71f9219a..377c16c761a66b2a0dac56630631e2f88823c7ce 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/RestoreStorageTask.java +++ b/o2server/x_console/src/main/java/com/x/server/console/RestoreStorageTask.java @@ -1,29 +1,29 @@ -package com.x.server.console; - -import com.x.base.core.project.config.Config; -import com.x.base.core.project.logger.Logger; -import com.x.base.core.project.logger.LoggerFactory; -import com.x.server.console.action.RestoreStorage; - -import org.quartz.Job; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; - -public class RestoreStorageTask implements Job { - - private static Logger logger = LoggerFactory.getLogger(RestoreStorageTask.class); - - @Override - public void execute(JobExecutionContext arg0) throws JobExecutionException { - try { - logger.print("schedule restore storage task start, restore from:{}.", - Config.currentNode().restoreData().path()); - RestoreStorage action = new RestoreStorage(); - action.execute(Config.currentNode().restoreStorage().path()); - } catch (Exception e) { - throw new JobExecutionException(e); - } - - } - -} \ No newline at end of file +//package com.x.server.console; +// +//import com.x.base.core.project.config.Config; +//import com.x.base.core.project.logger.Logger; +//import com.x.base.core.project.logger.LoggerFactory; +//import com.x.server.console.action.RestoreStorage; +// +//import org.quartz.Job; +//import org.quartz.JobExecutionContext; +//import org.quartz.JobExecutionException; +// +//public class RestoreStorageTask implements Job { +// +// private static Logger logger = LoggerFactory.getLogger(RestoreStorageTask.class); +// +// @Override +// public void execute(JobExecutionContext arg0) throws JobExecutionException { +// try { +// logger.print("schedule restore storage task start, restore from:{}.", +// Config.currentNode().restoreData().path()); +// RestoreStorage action = new RestoreStorage(); +// action.execute(Config.currentNode().restoreStorage().path()); +// } catch (Exception e) { +// throw new JobExecutionException(e); +// } +// +// } +// +//} \ No newline at end of file diff --git a/o2server/x_console/src/main/java/com/x/server/console/SchedulerBuilder.java b/o2server/x_console/src/main/java/com/x/server/console/SchedulerBuilder.java index 8a82d6a18a9ea685b42dc7e128d2be84411fa3db..73e2e39e7ce720256d65eb366c9e7e528fbbeb04 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/SchedulerBuilder.java +++ b/o2server/x_console/src/main/java/com/x/server/console/SchedulerBuilder.java @@ -28,14 +28,14 @@ public class SchedulerBuilder { .withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpData().cron())).build(); scheduler.scheduleJob(jobDetail, trigger); } - if (Config.currentNode().dumpStorage().enable() && Config.currentNode().dumpStorage().available()) { - JobDetail jobDetail = JobBuilder.newJob(DumpStorageTask.class) - .withIdentity(DumpStorageTask.class.getName(), scheduleGroup).withDescription(Config.node()) - .build(); - Trigger trigger = TriggerBuilder.newTrigger().withIdentity(DumpStorageTask.class.getName(), scheduleGroup) - .withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpStorage().cron())).build(); - scheduler.scheduleJob(jobDetail, trigger); - } +// if (Config.currentNode().dumpStorage().enable() && Config.currentNode().dumpStorage().available()) { +// JobDetail jobDetail = JobBuilder.newJob(DumpStorageTask.class) +// .withIdentity(DumpStorageTask.class.getName(), scheduleGroup).withDescription(Config.node()) +// .build(); +// Trigger trigger = TriggerBuilder.newTrigger().withIdentity(DumpStorageTask.class.getName(), scheduleGroup) +// .withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpStorage().cron())).build(); +// scheduler.scheduleJob(jobDetail, trigger); +// } if (Config.currentNode().restoreData().enable() && Config.currentNode().restoreData().available()) { JobDetail jobDetail = JobBuilder.newJob(RestoreDataTask.class) .withIdentity(RestoreDataTask.class.getName(), scheduleGroup).withDescription(Config.node()) @@ -44,16 +44,16 @@ public class SchedulerBuilder { .withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreData().cron())).build(); scheduler.scheduleJob(jobDetail, trigger); } - if (Config.currentNode().restoreStorage().enable() && Config.currentNode().restoreStorage().available()) { - JobDetail jobDetail = JobBuilder.newJob(RestoreStorageTask.class) - .withIdentity(RestoreStorageTask.class.getName(), scheduleGroup).withDescription(Config.node()) - .build(); - Trigger trigger = TriggerBuilder.newTrigger() - .withIdentity(RestoreStorageTask.class.getName(), scheduleGroup) - .withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreStorage().cron())) - .build(); - scheduler.scheduleJob(jobDetail, trigger); - } +// if (Config.currentNode().restoreStorage().enable() && Config.currentNode().restoreStorage().available()) { +// JobDetail jobDetail = JobBuilder.newJob(RestoreStorageTask.class) +// .withIdentity(RestoreStorageTask.class.getName(), scheduleGroup).withDescription(Config.node()) +// .build(); +// Trigger trigger = TriggerBuilder.newTrigger() +// .withIdentity(RestoreStorageTask.class.getName(), scheduleGroup) +// .withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreStorage().cron())) +// .build(); +// scheduler.scheduleJob(jobDetail, trigger); +// } this.registApplicationsAndVoteCenterTask(scheduler, scheduleGroup); return scheduler; diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/ActionControl.java b/o2server/x_console/src/main/java/com/x/server/console/action/ActionControl.java index ba8fd23cb37e3fa1c3ad81bab0f4b3bbce53a767..e9c0f510ea03dfdd34c0f5c428c77bfc2b914fe2 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/action/ActionControl.java +++ b/o2server/x_console/src/main/java/com/x/server/console/action/ActionControl.java @@ -33,9 +33,9 @@ public class ActionControl extends ActionBase { private static final String CMD_TD = "td"; private static final String CMD_EC = "ec"; private static final String CMD_DD = "dd"; - private static final String CMD_DS = "ds"; + // private static final String CMD_DS = "ds"; private static final String CMD_RD = "rd"; - private static final String CMD_RS = "rs"; + // private static final String CMD_RS = "rs"; private static final String CMD_CLH2 = "clh2"; private static final String CMD_UF = "uf"; private static final String CMD_DDL = "ddl"; @@ -63,12 +63,12 @@ public class ActionControl extends ActionBase { ec(cmd); } else if (cmd.hasOption(CMD_DD)) { dd(cmd); - } else if (cmd.hasOption(CMD_DS)) { - ds(cmd); +// } else if (cmd.hasOption(CMD_DS)) { +// ds(cmd); } else if (cmd.hasOption(CMD_RD)) { rd(cmd); - } else if (cmd.hasOption(CMD_RS)) { - rs(cmd); +// } else if (cmd.hasOption(CMD_RS)) { +// rs(cmd); } else if (cmd.hasOption(CMD_CLH2)) { clh2(cmd); } else if (cmd.hasOption(CMD_UF)) { @@ -96,9 +96,9 @@ public class ActionControl extends ActionBase { options.addOption(tdOption()); options.addOption(ecOption()); options.addOption(ddOption()); - options.addOption(dsOption()); + // options.addOption(dsOption()); options.addOption(rdOption()); - options.addOption(rsOption()); + // options.addOption(rsOption()); options.addOption(clh2Option()); options.addOption(ufOption()); options.addOption(ddlOption()); @@ -145,20 +145,20 @@ public class ActionControl extends ActionBase { .desc("导出数据库服务器的数据转换成json格式保存到本地文件.").build(); } - private static Option dsOption() { - return Option.builder("ds").longOpt("dumpStorage").argName("path").hasArg().optionalArg(true) - .desc("导出存储服务器的文件数据转换成json格式保存到本地文件.").build(); - } +// private static Option dsOption() { +// return Option.builder("ds").longOpt("dumpStorage").argName("path").hasArg().optionalArg(true) +// .desc("导出存储服务器的文件数据转换成json格式保存到本地文件.").build(); +// } private static Option rdOption() { return Option.builder("rd").longOpt("restoreData").argName("path or date").hasArg() .desc("将导出的json格式数据恢复到数据库服务器.").build(); } - private static Option rsOption() { - return Option.builder("rs").longOpt("restoreStorage").argName("path or date").hasArg() - .desc("将导出的json格式文件数据恢复到存储服务器.").build(); - } +// private static Option rsOption() { +// return Option.builder("rs").longOpt("restoreStorage").argName("path or date").hasArg() +// .desc("将导出的json格式文件数据恢复到存储服务器.").build(); +// } private static Option ufOption() { return Option.builder("uf").longOpt("updateFile").argName("path").hasArg().desc("升级服务器,升级前请注意备份.").build(); @@ -229,11 +229,11 @@ public class ActionControl extends ActionBase { dumpData.execute(path); } - private void ds(CommandLine cmd) throws Exception { - String path = Objects.toString(cmd.getOptionValue(CMD_DS), ""); - DumpStorage dumpStorage = new DumpStorage(); - dumpStorage.execute(path); - } +// private void ds(CommandLine cmd) throws Exception { +// String path = Objects.toString(cmd.getOptionValue(CMD_DS), ""); +// DumpStorage dumpStorage = new DumpStorage(); +// dumpStorage.execute(path); +// } private void rd(CommandLine cmd) throws Exception { String path = Objects.toString(cmd.getOptionValue(CMD_RD), ""); @@ -241,11 +241,11 @@ public class ActionControl extends ActionBase { restoreData.execute(path); } - private void rs(CommandLine cmd) throws Exception { - String path = Objects.toString(cmd.getOptionValue(CMD_RS), ""); - RestoreStorage restoreStorage = new RestoreStorage(); - restoreStorage.execute(path); - } +// private void rs(CommandLine cmd) throws Exception { +// String path = Objects.toString(cmd.getOptionValue(CMD_RS), ""); +// RestoreStorage restoreStorage = new RestoreStorage(); +// restoreStorage.execute(path); +// } private void hs(CommandLine cmd) { final Integer repeat = this.getArgInteger(cmd, CMD_HS, 1); diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java b/o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java index cc2989b3a986f542318653e60a1332f7f91f60ff..58a70dbc6e1b2314c9742af09bed9c4b9bfd19a6 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java +++ b/o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java @@ -1,12 +1,16 @@ package com.x.server.console.action; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; @@ -16,152 +20,198 @@ import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; import org.apache.commons.beanutils.BeanUtils; -import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.file.PathUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.openjpa.persistence.OpenJPAPersistence; import com.google.gson.Gson; import com.x.base.core.container.factory.PersistenceXmlHelper; import com.x.base.core.entity.JpaObject; +import com.x.base.core.entity.StorageObject; import com.x.base.core.entity.annotation.ContainerEntity; import com.x.base.core.entity.annotation.ContainerEntity.Reference; import com.x.base.core.project.config.Config; import com.x.base.core.project.config.DumpRestoreData; +import com.x.base.core.project.config.StorageMapping; +import com.x.base.core.project.config.StorageMappings; import com.x.base.core.project.gson.XGsonBuilder; import com.x.base.core.project.logger.Logger; import com.x.base.core.project.logger.LoggerFactory; import com.x.base.core.project.tools.DateTools; -import com.x.base.core.project.tools.DefaultCharset; import com.x.base.core.project.tools.ListTools; public class DumpData { private static Logger logger = LoggerFactory.getLogger(DumpData.class); - private Date start = new Date(); - - private File dir; - - private DumpRestoreDataCatalog catalog; - - private Gson pureGsonDateFormated = XGsonBuilder.instance(); - - @SuppressWarnings("unchecked") public boolean execute(String path) throws Exception { + Path dir = null; + Date start = new Date(); if (StringUtils.isEmpty(path)) { - this.dir = new File(Config.base(), "local/dump/dumpData_" + DateTools.compact(this.start)); + dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + DateTools.compact(start)); } else { - this.dir = new File(path); - if (dir.getAbsolutePath().startsWith(Config.base())) { - logger.print("path can not in base directory."); + dir = Paths.get(path); + if (dir.startsWith(Paths.get(Config.base()))) { + logger.warn("path can not in base directory."); return false; } } - FileUtils.forceMkdir(this.dir); - this.catalog = new DumpRestoreDataCatalog(); - - /* 初始化完成 */ - - List containerEntityNames = this.entities(); - - /** 过滤include exclude 条件 */ - List classNames = ListTools.includesExcludesWildcard(containerEntityNames, - Config.dumpRestoreData().getIncludes(), Config.dumpRestoreData().getExcludes()); - logger.print("dump data find {} data to dump, start at {}.", classNames.size(), DateTools.format(start)); - File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml"); - PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames); - for (int i = 0; i < classNames.size(); i++) { - Class cls = (Class) Class.forName(classNames.get(i)); - EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), - persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable())); - EntityManager em = emf.createEntityManager(); - try { - long estimateCount = this.estimateCount(em, cls); - logger.print("dump data({}/{}): {}, count: {}.", (i + 1), classNames.size(), cls.getName(), - estimateCount); - this.dump(cls, em, estimateCount); - } finally { - em.close(); - emf.close(); - } - } - FileUtils.write(new File(dir, "catalog.json"), pureGsonDateFormated.toJson(this.catalog), - DefaultCharset.charset); - logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.getAbsolutePath(), - this.count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60); + Files.createDirectories(dir); + Thread thread = new Thread(new RunnableImpl(dir, start)); + thread.start(); return true; } - private long estimateCount(EntityManager em, Class cls) { - CriteriaBuilder cb = em.getCriteriaBuilder(); - CriteriaQuery cq = cb.createQuery(Long.class); - Root root = cq.from(cls); - cq.select(cb.count(root)); - return em.createQuery(cq).getSingleResult(); - } + public class RunnableImpl implements Runnable { - private Integer count() { - return this.catalog.values().stream().mapToInt(Integer::intValue).sum(); - } + private Path dir; + private Date start; + private DumpRestoreDataCatalog catalog; + private Gson pureGsonDateFormated; + + public RunnableImpl(Path dir, Date start) { + this.dir = dir; + this.start = start; + this.catalog = new DumpRestoreDataCatalog(); + this.pureGsonDateFormated = XGsonBuilder.instance(); + } - private void dump(Class cls, EntityManager em, long total) - throws IOException, IllegalAccessException, InvocationTargetException, NoSuchMethodException { - /** 创建最终存储文件的目录 */ - File directory = new File(dir, cls.getName()); - FileUtils.forceMkdir(directory); - FileUtils.cleanDirectory(directory); - ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class); - String id = ""; - List list = null; - int count = 0; - int loop = 1; - int btach = (int) ((total + 0.0) / containerEntity.dumpSize()); - do { - list = this.list(em, cls, id, containerEntity.dumpSize()); - if (ListTools.isNotEmpty(list)) { - count = count + list.size(); - id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME); - File file = new File(directory, count + ".json"); - FileUtils.write(file, pureGsonDateFormated.toJson(list), DefaultCharset.charset); - logger.print("dumping {}/{} part of data:{}.", loop++, btach, cls.getName()); + public void run() { + try { + List classNames = entities(); + logger.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start)); + Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(), + DateTools.compact(start) + "_dump.xml"); + PersistenceXmlHelper.write(xml.toString(), classNames); + StorageMappings storageMappings = Config.storageMappings(); + Stream stream = BooleanUtils.isTrue(Config.dumpRestoreData().getParallel()) + ? classNames.parallelStream() + : classNames.stream(); + AtomicInteger idx = new AtomicInteger(1); + stream.forEach(className -> { + String nameOfThread = Thread.currentThread().getName(); + EntityManagerFactory emf = null; + EntityManager em = null; + try { + Thread.currentThread().setName(DumpData.class.getName() + ":" + className); + @SuppressWarnings("unchecked") + Class cls = (Class) Class.forName(className); + emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(), + PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable())); + em = emf.createEntityManager(); + long estimateCount = estimateCount(em, cls); + logger.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(), + cls.getName(), estimateCount); + dump(cls, em, storageMappings, estimateCount); + } catch (Exception e) { + logger.error(new Exception(String.format("dump:%s error.", className), e)); + } finally { + Thread.currentThread().setName(nameOfThread); + em.close(); + emf.close(); + } + }); + Files.write(dir.resolve("catalog.json"), + pureGsonDateFormated.toJson(catalog).getBytes(StandardCharsets.UTF_8)); + logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(), + count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60); + } catch (Exception e) { + e.printStackTrace(); } - em.clear(); - } while (ListTools.isNotEmpty(list)); - this.catalog.put(cls.getName(), count); - } + } + + @SuppressWarnings("unchecked") + private List entities() throws Exception { + List list = new ArrayList<>(); + if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) { + list.addAll((List) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)); + return list; + } + for (String str : (List) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)) { + Class cls = Class.forName(str); + ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class); + if (Objects.equals(containerEntity.reference(), Reference.strong)) { + list.add(str); + } + } + return ListTools.includesExcludesWildcard(list, Config.dumpRestoreData().getIncludes(), + Config.dumpRestoreData().getExcludes()); + } - private List list(EntityManager em, Class cls, String id, Integer size) { - CriteriaBuilder cb = em.getCriteriaBuilder(); - CriteriaQuery cq = cb.createQuery(cls); - Root root = cq.from(cls); - Predicate p = cb.conjunction(); - if (StringUtils.isNotEmpty(id)) { - p = cb.greaterThan(root.get(JpaObject.id_FIELDNAME), id); + private long estimateCount(EntityManager em, Class cls) { + CriteriaBuilder cb = em.getCriteriaBuilder(); + CriteriaQuery cq = cb.createQuery(Long.class); + Root root = cq.from(cls); + cq.select(cb.count(root)); + return em.createQuery(cq).getSingleResult(); } - cq.select(root).where(p).orderBy(cb.asc(root.get(JpaObject.id_FIELDNAME))); - return em.createQuery(cq).setMaxResults(size).getResultList(); - } - /** - * 根据设置的模式不同输出需要dump的entity className - * - * @return - * @throws Exception - */ - @SuppressWarnings("unchecked") - private List entities() throws Exception { - List list = new ArrayList<>(); - if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) { - list.addAll((List) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)); - return list; + private Integer count() { + return catalog.values().stream().mapToInt(Integer::intValue).sum(); } - for (String str : (List) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)) { - Class cls = Class.forName(str); + + private List list(EntityManager em, Class cls, String id, Integer size) { + CriteriaBuilder cb = em.getCriteriaBuilder(); + CriteriaQuery cq = cb.createQuery(cls); + Root root = cq.from(cls); + Predicate p = cb.conjunction(); + if (StringUtils.isNotEmpty(id)) { + p = cb.greaterThan(root.get(JpaObject.id_FIELDNAME), id); + } + cq.select(root).where(p).orderBy(cb.asc(root.get(JpaObject.id_FIELDNAME))); + return em.createQuery(cq).setMaxResults(size).getResultList(); + } + + private void dump(Class cls, EntityManager em, StorageMappings storageMappings, long total) + throws Exception { + // 创建最终存储文件的目录 + Path directory = dir.resolve(cls.getName()); + Files.createDirectories(directory); + PathUtils.cleanDirectory(directory); ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class); - if (Objects.equals(containerEntity.reference(), Reference.strong)) { - list.add(str); + String id = ""; + List list = null; + int count = 0; + int loop = 1; + int btach = (int) Math.ceil((total + 0.0) / containerEntity.dumpSize()); + do { + list = list(em, cls, id, containerEntity.dumpSize()); + if (ListTools.isNotEmpty(list)) { + count = count + list.size(); + id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME); + if (StorageObject.class.isAssignableFrom(cls)) { + Path sub = directory.resolve(count + ""); + Files.createDirectories(sub); + binary(list, sub, storageMappings); + } + Files.write(directory.resolve(count + ".json"), + pureGsonDateFormated.toJson(list).getBytes(StandardCharsets.UTF_8)); + logger.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName()); + } + em.clear(); + } while (ListTools.isNotEmpty(list)); + catalog.put(cls.getName(), count); + } + + private void binary(List list, Path sub, StorageMappings storageMappings) throws Exception { + for (T t : list) { + StorageObject s = (StorageObject) t; + String name = s.getStorage(); + StorageMapping mapping = storageMappings.get(s.getClass(), name); + if (null == mapping && Config.dumpRestoreStorage().getExceptionInvalidStorage()) { + throw new ExceptionInvalidStorage(s); + } + if (null != mapping) { + Path p = sub.resolve(FilenameUtils.getName(s.path())); + if (s.existContent(mapping)) { + try (OutputStream out = Files.newOutputStream(p)) { + out.write(s.readContent(mapping)); + } + } + } } } - return list; } } \ No newline at end of file diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/DumpStorage.java b/o2server/x_console/src/main/java/com/x/server/console/action/DumpStorage.java deleted file mode 100644 index 04a2235e2af61e9b89029a4cb4b8c695cc27e7bb..0000000000000000000000000000000000000000 --- a/o2server/x_console/src/main/java/com/x/server/console/action/DumpStorage.java +++ /dev/null @@ -1,261 +0,0 @@ -package com.x.server.console.action; - -import java.io.File; -import java.io.FileOutputStream; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Objects; - -import javax.persistence.EntityManager; -import javax.persistence.EntityManagerFactory; -import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.Predicate; -import javax.persistence.criteria.Root; - -import org.apache.commons.beanutils.BeanUtils; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.openjpa.persistence.OpenJPAPersistence; - -import com.google.gson.Gson; -import com.x.base.core.container.factory.PersistenceXmlHelper; -import com.x.base.core.entity.JpaObject; -import com.x.base.core.entity.StorageObject; -import com.x.base.core.entity.annotation.ContainerEntity; -import com.x.base.core.entity.annotation.ContainerEntity.Reference; -import com.x.base.core.project.config.Config; -import com.x.base.core.project.config.DumpRestoreData; -import com.x.base.core.project.config.StorageMapping; -import com.x.base.core.project.config.StorageMappings; -import com.x.base.core.project.gson.XGsonBuilder; -import com.x.base.core.project.logger.Logger; -import com.x.base.core.project.logger.LoggerFactory; -import com.x.base.core.project.tools.DateTools; -import com.x.base.core.project.tools.DefaultCharset; -import com.x.base.core.project.tools.ListTools; - -public class DumpStorage { - - private static Logger logger = LoggerFactory.getLogger(DumpStorage.class); - - private Date start = new Date(); - - private File dir; - - private DumpRestoreStorageCatalog catalog; - - private Gson pureGsonDateFormated = XGsonBuilder.instance(); - - public boolean execute(String path) throws Exception { - if (StringUtils.isEmpty(path)) { - this.dir = new File(Config.base(), "local/dump/dumpStorage_" + DateTools.compact(this.start)); - } else { - this.dir = new File(path); - if (dir.getAbsolutePath().startsWith(Config.base())) { - logger.print("path can not in base directory."); - return false; - } - } - FileUtils.forceMkdir(this.dir); - this.catalog = new DumpRestoreStorageCatalog(); - - List storageContainerEntityNames = this.entities(); - List classNames = ListTools.includesExcludesWildcard(storageContainerEntityNames, - Config.dumpRestoreStorage().getIncludes(), Config.dumpRestoreStorage().getExcludes()); - logger.print("dump storage find {} data to dump, start at {}.", classNames.size(), DateTools.format(start)); - StorageMappings storageMappings = Config.storageMappings(); - File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml"); - PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames); - for (int i = 0; i < classNames.size(); i++) { - Class cls = (Class) Class.forName(classNames.get(i)); - EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), - persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable())); - EntityManager em = emf.createEntityManager(); - try { - logger.print("dump storage({}/{}): {}, estimate count: {}, estimate size: {}M.", (i + 1), - classNames.size(), cls.getName(), this.estimateCount(em, cls), - (this.estimateSize(em, cls) / 1024 / 1024)); - this.dump(cls, em, storageMappings); - } finally { - em.close(); - emf.close(); - } - } - FileUtils.write(new File(dir, "catalog.json"), XGsonBuilder.instance().toJson(this.catalog), - DefaultCharset.charset); - logger.print( - "dump storage completed, directory: {}, count: {}, normal: {}, empty: {}, invalidStorage: {}, size: {}M, elapsed: {} minutes.", - dir.getAbsolutePath(), this.count(), this.normal(), this.empty(), this.invalidStorage(), - (this.size() / 1024 / 1024), (System.currentTimeMillis() - start.getTime()) / 1000 / 60); - return true; - } - - private Integer count() { - return this.catalog.values().stream().mapToInt(DumpRestoreStorageCatalogItem::getCount).sum(); - } - - private Long size() { - return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getSize).sum(); - } - - private Long normal() { - return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getNormal).sum(); - } - - private Long empty() { - return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getEmpty).sum(); - } - - private Long invalidStorage() { - return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getInvalidStorage).sum(); - } - - private long estimateCount(EntityManager em, Class cls) { - CriteriaBuilder cb = em.getCriteriaBuilder(); - CriteriaQuery cq = cb.createQuery(Long.class); - Root root = cq.from(cls); - cq.select(cb.count(root)); - return em.createQuery(cq).getSingleResult(); - } - - private long estimateSize(EntityManager em, Class cls) { - CriteriaBuilder cb = em.getCriteriaBuilder(); - CriteriaQuery cq = cb.createQuery(Long.class); - Root root = cq.from(cls); - cq.select(root.get("length")); - List list = em.createQuery(cq).getResultList(); - if (!list.isEmpty()) { - /** 上面的语句有可能返回的是null值,所以要先过滤 */ - return list.stream().filter(o -> null != o).mapToLong(Long::longValue).sum(); - } else { - return 0L; - } - } - - private void dump(Class cls, EntityManager em, StorageMappings storageMappings) - throws Exception { - /** 创建最终存储文件的目录 */ - File classDirectory = new File(dir, cls.getName()); - FileUtils.forceMkdir(classDirectory); - FileUtils.cleanDirectory(classDirectory); - int count = 0; - long size = 0L; - int normal = 0; - int invalidStorage = 0; - int empty = 0; - String id = ""; - String name = ""; - List list = null; - File directory = null; - StorageMapping mapping = null; - List normalList = null; - List emptyList = null; - List invalidStorageList = null; - ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class); - do { - list = this.list(em, cls, id, containerEntity.dumpSize()); - if (ListTools.isNotEmpty(list)) { - count += list.size(); - directory = new File(classDirectory, Integer.toString(count)); - FileUtils.forceMkdir(directory); - FileUtils.cleanDirectory(directory); - normalList = new ArrayList(); - emptyList = new ArrayList(); - invalidStorageList = new ArrayList(); - for (T t : list) { - name = t.getStorage(); - mapping = storageMappings.get(cls, name); - if (StringUtils.isNotEmpty(name)) { - if (null == mapping && Config.dumpRestoreStorage().getExceptionInvalidStorage()) { - throw new Exception("can not find storageMapping class: " + cls.getName() + ", storage: " - + name + ", id: " + t.getId() + ", name: " + t.getName() - + ", set exceptionInvalidStorage to false will ignore item."); - } - } - if (null != mapping) { - File file = new File(directory, FilenameUtils.getName(t.path())); - if (t.existContent(mapping)) { - try (FileOutputStream fos = new FileOutputStream(file)) { - size += t.readContent(mapping, fos); - normalList.add(t); - normal++; - } - } else { - emptyList.add(t); - empty++; - } - } else { - invalidStorageList.add(t); - invalidStorage++; - } - } - id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME); - File file = new File(classDirectory, count + ".json"); - this.dumpWrite(file, normalList, emptyList, invalidStorageList); - } - em.clear(); - } while (ListTools.isNotEmpty(list)); - DumpRestoreStorageCatalogItem item = new DumpRestoreStorageCatalogItem(); - item.setCount(count); - item.setNormal(normal); - item.setEmpty(empty); - item.setSize(size); - item.setInvalidStorage(invalidStorage); - this.catalog.put(cls.getName(), item); - logger.print( - "dumped storage: " + cls.getName() + ", count: " + count + ", normal: " + normal + ", invalidStorage: " - + invalidStorage + ", empty: " + empty + ", size: " + (size / 1024 / 1024) + "M."); - } - - private void dumpWrite(File file, List normalList, List emptyList, - List invalidStorageList) throws Exception { - LinkedHashMap> o = new LinkedHashMap<>(); - o.put("normals", normalList); - o.put("emptys", emptyList); - o.put("invalidStorages", invalidStorageList); - FileUtils.write(file, pureGsonDateFormated.toJson(o), DefaultCharset.charset); - } - - private List list(EntityManager em, Class cls, String id, Integer size) { - CriteriaBuilder cb = em.getCriteriaBuilder(); - CriteriaQuery cq = cb.createQuery(cls); - Root root = cq.from(cls); - Predicate p = cb.conjunction(); - if (StringUtils.isNotEmpty(id)) { - p = cb.greaterThan(root.get("id"), id); - } - cq.select(root).where(p).orderBy(cb.asc(root.get("id"))); - return em.createQuery(cq).setMaxResults(size).getResultList(); - } - - /** - * 根据设置的模式不同输出需要dump的entity className - * - * @return - * @throws Exception - */ - @SuppressWarnings("unchecked") - private List entities() throws Exception { - List list = new ArrayList<>(); - if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) { - list.addAll((List) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES)); - return list; - } - for (String str : (List) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES)) { - Class cls = Class.forName(str); - ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class); - if (Objects.equals(containerEntity.reference(), Reference.strong)) { - list.add(str); - } - } - return list; - } - - public static class Item { - - } -} \ No newline at end of file diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionDirectoryNotExist.java b/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionDirectoryNotExist.java new file mode 100644 index 0000000000000000000000000000000000000000..f49f129740ffb75585a6e1fb5591a873b241fcb5 --- /dev/null +++ b/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionDirectoryNotExist.java @@ -0,0 +1,15 @@ +package com.x.server.console.action; + +import java.nio.file.Path; + +import com.x.base.core.project.exception.PromptException; + +class ExceptionDirectoryNotExist extends PromptException { + + private static final long serialVersionUID = -5515077418025884395L; + + ExceptionDirectoryNotExist(Path path) { + super("directory not exist: {}.", path.toString()); + } + +} diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionFileNotExist.java b/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionFileNotExist.java new file mode 100644 index 0000000000000000000000000000000000000000..cfdd2169b81d62cb887bddc4dd798bdd802c7bff --- /dev/null +++ b/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionFileNotExist.java @@ -0,0 +1,16 @@ +package com.x.server.console.action; + + +import java.nio.file.Path; + +import com.x.base.core.project.exception.PromptException; + +class ExceptionFileNotExist extends PromptException { + + private static final long serialVersionUID = -5515077418025884395L; + + ExceptionFileNotExist(Path path) { + super("file not exist: {}.", path.toString()); + } + +} diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionInvalidStorage.java b/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionInvalidStorage.java new file mode 100644 index 0000000000000000000000000000000000000000..21275660e91a24c1c05b24f92437fba96351172b --- /dev/null +++ b/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionInvalidStorage.java @@ -0,0 +1,16 @@ +package com.x.server.console.action; + +import com.x.base.core.entity.StorageObject; +import com.x.base.core.project.exception.PromptException; + +class ExceptionInvalidStorage extends PromptException { + + private static final long serialVersionUID = -5515077418025884395L; + + ExceptionInvalidStorage(StorageObject storageObject) { + super("can not find storageMapping class: " + storageObject.getClass().getName() + ", storage: " + + storageObject.getStorage() + ", id: " + storageObject.getId() + ", name: " + storageObject.getName() + + ", set exceptionInvalidStorage to false will ignore item."); + } + +} diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionMappingNotExist.java b/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionMappingNotExist.java new file mode 100644 index 0000000000000000000000000000000000000000..13038869914c5e03acb066b0974ec2c9099ea6ab --- /dev/null +++ b/o2server/x_console/src/main/java/com/x/server/console/action/ExceptionMappingNotExist.java @@ -0,0 +1,13 @@ +package com.x.server.console.action; + +import com.x.base.core.project.exception.PromptException; + +class ExceptionMappingNotExist extends PromptException { + + private static final long serialVersionUID = -5515077418025884395L; + + ExceptionMappingNotExist() { + super("can not find storageMapping"); + } + +} diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java b/o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java index 03f5bd5a7f2605efc742922699ed26fcf912bd60..4f6b7035b2f7ef984caa794c235dad9dfaa72ad6 100644 --- a/o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java +++ b/o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java @@ -1,176 +1,249 @@ package com.x.server.console.action; -import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.Date; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; import javax.persistence.EntityManager; import javax.persistence.EntityManagerFactory; -import javax.persistence.FlushModeType; import javax.persistence.criteria.CriteriaBuilder; import javax.persistence.criteria.CriteriaQuery; import javax.persistence.criteria.Root; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.openjpa.persistence.OpenJPAPersistence; + import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.x.base.core.container.factory.PersistenceXmlHelper; import com.x.base.core.entity.JpaObject; +import com.x.base.core.entity.StorageObject; +import com.x.base.core.entity.annotation.ContainerEntity; import com.x.base.core.project.config.Config; +import com.x.base.core.project.config.StorageMapping; +import com.x.base.core.project.config.StorageMappings; import com.x.base.core.project.gson.XGsonBuilder; import com.x.base.core.project.logger.Logger; import com.x.base.core.project.logger.LoggerFactory; -import com.x.base.core.project.tools.BaseTools; import com.x.base.core.project.tools.DateTools; -import com.x.base.core.project.tools.DefaultCharset; import com.x.base.core.project.tools.ListTools; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.openjpa.persistence.OpenJPAPersistence; - -/** - * @author zhourui - */ public class RestoreData { private static Logger logger = LoggerFactory.getLogger(RestoreData.class); - private Date start = new Date(); - - private File dir; - - private DumpRestoreDataCatalog catalog; - - private Gson pureGsonDateFormated = XGsonBuilder.instance(); - public boolean execute(String path) throws Exception { + Date start = new Date(); + Path dir; if (StringUtils.isEmpty(path)) { - logger.print("path is empty."); + logger.warn("path is empty."); } if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) { - this.dir = new File(Config.base(), "local/dump/dumpData_" + path); - this.catalog = BaseTools.readConfigObject("local/dump/dumpData_" + path + "/catalog.json", - DumpRestoreDataCatalog.class); + dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + path); } else { - this.dir = new File(path); - if (!(this.dir.exists() && this.dir.isDirectory())) { - logger.print("dir not exist: {}.", path); + dir = Paths.get(path); + if ((!Files.exists(dir)) || (!Files.isDirectory(dir))) { + logger.warn("directory not exist: {}.", path); return false; - } else if (StringUtils.startsWith(dir.getAbsolutePath(), Config.base())) { - logger.print("path can not in base directory."); + } else if (dir.startsWith(Paths.get(Config.base()))) { + logger.warn("path can not in base directory."); return false; } } - this.catalog = XGsonBuilder.instance().fromJson( - FileUtils.readFileToString(new File(dir, "catalog.json"), DefaultCharset.charset_utf_8), - DumpRestoreDataCatalog.class); - return this.execute(); + Thread thread = new Thread(new RunnableImpl(dir, start)); + thread.start(); + return true; } - @SuppressWarnings("unchecked") - public boolean execute() throws Exception { - List containerEntityNames = new ArrayList<>(); - containerEntityNames.addAll((List) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)); - List classNames = new ArrayList<>(); - classNames.addAll(this.catalog.keySet()); - classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(), - Config.dumpRestoreData().getExcludes()); - classNames = ListTools.includesExcludesWildcard(containerEntityNames, classNames, null); - - logger.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.getAbsolutePath()); - File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml"); - PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames); - long count = 0; - for (int i = 0; i < classNames.size(); i++) { - Class cls = (Class) Class.forName(classNames.get(i)); + public class RunnableImpl implements Runnable { + + private Path dir; + private Date start; + private DumpRestoreDataCatalog catalog; + private Gson gson; + + public RunnableImpl(Path dir, Date start) throws IOException { + this.dir = dir; + this.start = start; + this.catalog = new DumpRestoreDataCatalog(); + this.gson = XGsonBuilder.instance(); + Path path = dir.resolve("catalog.json"); + this.catalog = XGsonBuilder.instance().fromJson( + new String(Files.readAllBytes(path), StandardCharsets.UTF_8), DumpRestoreDataCatalog.class); + } + + @Override + public void run() { + try { + List classNames = this.entities(); + logger.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.toString()); + Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(), + DateTools.compact(start) + "_restore.xml"); + PersistenceXmlHelper.write(xml.toString(), classNames); + Stream stream = BooleanUtils.isTrue(Config.dumpRestoreData().getParallel()) + ? classNames.parallelStream() + : classNames.stream(); + AtomicInteger idx = new AtomicInteger(1); + AtomicLong total = new AtomicLong(0); + stream.forEach(className -> { + String nameOfThread = Thread.currentThread().getName(); + try { + Thread.currentThread().setName(RestoreData.class.getName() + ":" + className); + @SuppressWarnings("unchecked") + Class cls = (Class) Class.forName(className); + logger.print("restore data({}/{}): {}.", idx.getAndAdd(1), classNames.size(), cls.getName()); + long size = restore(cls, xml); + total.getAndAdd(size); + } catch (Exception e) { + logger.error(new Exception(String.format("restore:%s error.", className), e)); + } finally { + Thread.currentThread().setName(nameOfThread); + } + }); + logger.print("restore data completed, directory: {}, count: {}, total: {}, elapsed: {} minutes.", + dir.toString(), idx.get(), total.longValue(), + (System.currentTimeMillis() - start.getTime()) / 1000 / 60); + } catch (Exception e) { + logger.error(e); + } + } + + @SuppressWarnings("unchecked") + private List entities() throws Exception { + List containerEntityNames = new ArrayList<>(); + containerEntityNames.addAll((List) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)); + List classNames = new ArrayList<>(); + classNames.addAll(this.catalog.keySet()); + classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(), + Config.dumpRestoreData().getExcludes()); + return ListTools.includesExcludesWildcard(containerEntityNames, classNames, null); + } + + private long restore(Class cls, Path xml) throws Exception { EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), - persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable())); - if (emf != null) { - EntityManager em = emf.createEntityManager(); - em.setFlushMode(FlushModeType.COMMIT); - try { - logger.print("restore data({}/{}): {}, count: {}.", (i + 1), classNames.size(), cls.getName(), - catalog.get(cls.getName())); - count = count + this.store(cls, em); - } finally { - em.close(); - emf.close(); + xml.getFileName().toString(), + PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable())); + AtomicLong count = new AtomicLong(0); + AtomicInteger batch = new AtomicInteger(1); + try { + Path directory = dir.resolve(cls.getName()); + if ((!Files.exists(directory)) || (!Files.isDirectory(directory))) { + throw new ExceptionDirectoryNotExist(directory); } - } else { - logger.warn("can not create 'EntityManagerFactory' for Entity:[" + cls.getName() + "]"); + StorageMappings storageMappings = Config.storageMappings(); + this.clean(cls, emf, storageMappings, cls.getAnnotation(ContainerEntity.class)); + List paths = this.list(directory); + paths.stream().forEachOrdered(o -> { + logger.print("restore {}/{} part of data:{}.", batch.getAndAdd(1), paths.size(), cls.getName()); + EntityManager em = null; + try { + em = emf.createEntityManager(); + em.getTransaction().begin(); + JsonArray raws = this.convert(o); + for (JsonElement json : raws) { + Object t = gson.fromJson(json, cls); + if (StorageObject.class.isAssignableFrom(cls)) { + Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString())); + this.binary(t, cls, sub, storageMappings); + } + em.persist(t); + count.getAndAdd(1); + } + em.getTransaction().commit(); + em.clear(); + } catch (Exception e) { + logger.error(new Exception(String.format("restore error with file:%s.", o.toString()), e)); + } finally { + em.close(); + } + }); + logger.print("restore data: {} completed, count: {}.", cls.getName(), count.intValue()); + } catch (Exception e) { + logger.error(e); + } finally { + emf.close(); } + return count.longValue(); } - logger.print("restore data completed, total count: {}, elapsed: {} minutes.", count, - (System.currentTimeMillis() - start.getTime()) / 1000 / 60); - return true; - } - private long store(Class cls, EntityManager em) throws Exception { - File directory = new File(this.dir, cls.getName()); - if ((!directory.exists()) || (!directory.isDirectory())) { - throw new Exception("can not find directory: " + directory.getAbsolutePath() + "."); + private List list(Path directory) throws IOException { + List list = new ArrayList<>(); + try (Stream stream = Files.list(directory)) { + stream.filter(p -> StringUtils.endsWithIgnoreCase(p.getFileName().toString(), ".json")) + .sorted((Path p1, Path p2) -> { + Integer i1 = Integer.parseInt(FilenameUtils.getBaseName(p1.getFileName().toString())); + Integer i2 = Integer.parseInt(FilenameUtils.getBaseName(p2.getFileName().toString())); + return i1.compareTo(i2); + }).forEach(list::add); + } + return list; } - long count = 0; - List files = new ArrayList<>(FileUtils.listFiles(directory, new String[] { "json" }, false)); - /** 对文件进行排序,和dump的时候的顺序保持一直 */ - Collections.sort(files, new Comparator() { - public int compare(File o1, File o2) { - String n1 = FilenameUtils.getBaseName(o1.getName()); - String n2 = FilenameUtils.getBaseName(o2.getName()); - Integer i1 = Integer.parseInt(n1); - Integer i2 = Integer.parseInt(n2); - return i1.compareTo(i2); + + @SuppressWarnings("unchecked") + private void binary(Object o, Class cls, Path sub, StorageMappings storageMappings) throws Exception { + StorageObject so = (StorageObject) o; + StorageMapping mapping = null; + if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistribute())) { + mapping = storageMappings.random((Class) cls); + } else { + mapping = storageMappings.get((Class) cls, so.getStorage()); + } + if (null == mapping) { + throw new ExceptionMappingNotExist(); + } + Path path = sub.resolve(Paths.get(so.path()).getFileName()); + if (!Files.exists(path)) { + throw new ExceptionFileNotExist(path); } - }); - /** 尽量在最后进行清空操作 */ - this.clean(cls, em); - File file = null; - for (int i = 0; i < files.size(); i++) { - file = files.get(i); - logger.print("restoring {}/{} part of data:{}.", (i + 1), files.size(), cls.getName()); - JsonArray raws = this.convert(file); - em.getTransaction().begin(); - for (JsonElement o : raws) { - T t = pureGsonDateFormated.fromJson(o, cls); - em.persist(t); - count++; + try (InputStream input = Files.newInputStream(path)) { + so.saveContent(mapping, input, so.getName()); } - em.getTransaction().commit(); - em.clear(); } - System.out.println("restore data: " + cls.getName() + " completed, count: " + count + "."); - return count; - - } - private JsonArray convert(File file) throws Exception { - /* 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double */ - String json = FileUtils.readFileToString(file, DefaultCharset.charset); - JsonElement jsonElement = pureGsonDateFormated.fromJson(json, JsonElement.class); - return jsonElement.getAsJsonArray(); - } + private JsonArray convert(Path path) throws IOException { + // 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double + String json = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); + JsonElement jsonElement = gson.fromJson(json, JsonElement.class); + return jsonElement.getAsJsonArray(); + } - private void clean(Class cls, EntityManager em) throws Exception { - List list = null; - do { - if (ListTools.isNotEmpty(list)) { - em.getTransaction().begin(); - for (T t : list) { - em.remove(t); + private void clean(Class cls, EntityManagerFactory emf, StorageMappings storageMappings, + ContainerEntity containerEntity) throws Exception { + EntityManager em = emf.createEntityManager(); + List list = null; + do { + if (ListTools.isNotEmpty(list)) { + em.getTransaction().begin(); + for (T t : list) { + em.remove(t); + if (StorageObject.class.isAssignableFrom(cls)) { + StorageObject so = (StorageObject) t; + @SuppressWarnings("unchecked") + StorageMapping mapping = storageMappings.get((Class) cls, so.getStorage()); + if (null != mapping) { + so.deleteContent(mapping); + } + } + } + em.getTransaction().commit(); } - em.getTransaction().commit(); - } - CriteriaBuilder cb = em.getCriteriaBuilder(); - CriteriaQuery cq = cb.createQuery(cls); - Root root = cq.from(cls); - cq.select(root); - list = em.createQuery(cq).setMaxResults(Config.dumpRestoreData().getBatchSize()).getResultList(); - } while (ListTools.isNotEmpty(list)); + CriteriaBuilder cb = em.getCriteriaBuilder(); + CriteriaQuery cq = cb.createQuery(cls); + Root root = cq.from(cls); + list = em.createQuery(cq.select(root)).setMaxResults(containerEntity.dumpSize()).getResultList(); + } while (ListTools.isNotEmpty(list)); + } } } \ No newline at end of file diff --git a/o2server/x_console/src/main/java/com/x/server/console/action/RestoreStorage.java b/o2server/x_console/src/main/java/com/x/server/console/action/RestoreStorage.java deleted file mode 100644 index 9162e70857a1ef5f0fb631aa3ac8e57fc5143929..0000000000000000000000000000000000000000 --- a/o2server/x_console/src/main/java/com/x/server/console/action/RestoreStorage.java +++ /dev/null @@ -1,206 +0,0 @@ -package com.x.server.console.action; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.List; - -import javax.persistence.EntityManager; -import javax.persistence.EntityManagerFactory; -import javax.persistence.FlushModeType; -import javax.persistence.criteria.CriteriaBuilder; -import javax.persistence.criteria.CriteriaQuery; -import javax.persistence.criteria.Root; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.openjpa.persistence.OpenJPAPersistence; - -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.x.base.core.container.factory.PersistenceXmlHelper; -import com.x.base.core.entity.StorageObject; -import com.x.base.core.project.config.Config; -import com.x.base.core.project.config.StorageMapping; -import com.x.base.core.project.config.StorageMappings; -import com.x.base.core.project.gson.XGsonBuilder; -import com.x.base.core.project.logger.Logger; -import com.x.base.core.project.logger.LoggerFactory; -import com.x.base.core.project.tools.BaseTools; -import com.x.base.core.project.tools.DateTools; -import com.x.base.core.project.tools.DefaultCharset; -import com.x.base.core.project.tools.ListTools; - -/** - * @author zhourui - */ - -public class RestoreStorage { - - private static Logger logger = LoggerFactory.getLogger(RestoreStorage.class); - - private Date start = new Date(); - - private File dir; - - private DumpRestoreStorageCatalog catalog; - - private Gson pureGsonDateFormated = XGsonBuilder.instance(); - - public boolean execute(final String path) throws Exception { - if (StringUtils.isEmpty(path)) { - logger.print("path is empty."); - } - if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) { - this.dir = new File(Config.base(), "local/dump/dumpStorage_" + path); - this.catalog = BaseTools.readConfigObject("local/dump/dumpStorage_" + path + "/catalog.json", - DumpRestoreStorageCatalog.class); - } else { - this.dir = new File(path); - if (!(this.dir.exists() && this.dir.isDirectory())) { - logger.print("dir not exist: {}.", path); - return false; - } else if (StringUtils.startsWith(dir.getAbsolutePath(), Config.base())) { - logger.print("path can not in base directory."); - return false; - } - this.catalog = XGsonBuilder.instance().fromJson(FileUtils - .readFileToString(new File(dir.getAbsolutePath(), "catalog.json"), DefaultCharset.charset_utf_8), - DumpRestoreStorageCatalog.class); - } - return this.execute(); - } - - private boolean execute() throws Exception { - final List storageContainerEntityNames = new ArrayList<>(); - storageContainerEntityNames.addAll((List) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES)); - List classNames = new ArrayList<>(); - classNames.addAll(this.catalog.keySet()); - classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreStorage().getIncludes(), - Config.dumpRestoreStorage().getExcludes()); - logger.print("restore storage find {} to restore.", classNames.size()); - final File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml"); - PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames); - final StorageMappings storageMappings = Config.storageMappings(); - int count = 0; - for (int i = 0; i < classNames.size(); i++) { - final Class cls = (Class) Class.forName(classNames.get(i)); - final EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), - persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable())); - final EntityManager em = emf.createEntityManager(); - em.setFlushMode(FlushModeType.COMMIT); - try { - final DumpRestoreStorageCatalogItem item = this.catalog.get(cls.getName()); - logger.print( - "restore storage({}/{}): {}, count: {}, normal: {} will be restore, invalidStorage: {} and empty: {} will be ignore, size: {}M.", - (i + 1), classNames.size(), cls.getName(), item.getCount(), item.getNormal(), - item.getInvalidStorage(), item.getEmpty(), (item.getSize() / 1024 / 1024)); - count += this.store(cls, em, storageMappings); - } finally { - em.close(); - emf.close(); - } - logger.print("restore storage completed, total count: {}, elapsed: {} minutes.", count, - (System.currentTimeMillis() - start.getTime()) / 1000 / 60); - } - return false; - } - - private long store(final Class cls, final EntityManager em, - final StorageMappings storageMappings) throws Exception { - final File classDirectory = new File(this.dir, cls.getName()); - if ((!classDirectory.exists()) || (!classDirectory.isDirectory())) { - throw new Exception("can not find directory: " + classDirectory.getAbsolutePath() + "."); - } - long count = 0; - final List files = new ArrayList( - FileUtils.listFiles(classDirectory, new String[] { "json" }, false)); - /** 对文件进行排序,和dump的时候的顺序保持一直 */ - Collections.sort(files, new Comparator() { - public int compare(final File o1, final File o2) { - final String n1 = FilenameUtils.getBaseName(o1.getName()); - final String n2 = FilenameUtils.getBaseName(o2.getName()); - final Integer i1 = Integer.parseInt(n1); - final Integer i2 = Integer.parseInt(n2); - return i1.compareTo(i2); - } - }); - /** 尽量将删除的工作放在后面 */ - this.clean(cls, em, storageMappings); - StorageMapping mapping = null; - File file = null; - for (int i = 0; i < files.size(); i++) { - file = files.get(i); - /** 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double */ - logger.print("restoring " + (i + 1) + "/" + files.size() + " part of storage: " + cls.getName() + "."); - final JsonArray raws = this.convert(file); - if (null != raws) { - em.getTransaction().begin(); - for (final JsonElement o : raws) { - final T t = pureGsonDateFormated.fromJson(o, cls); - if (Config.dumpRestoreStorage().getRedistribute()) { - mapping = storageMappings.random(cls); - } else { - mapping = storageMappings.get(cls, t.getStorage()); - } - if (null == mapping) { - throw new Exception( - "can not find storageMapping class: " + cls.getName() + ", name:" + t.getName()); - } - final File source = new File(classDirectory, FilenameUtils.getBaseName(file.getName()) - + StorageObject.PATHSEPARATOR + FilenameUtils.getName(t.path())); - try (FileInputStream input = new FileInputStream(source)) { - t.saveContent(mapping, input, t.getName()); - } - em.persist(t); - count++; - } - em.getTransaction().commit(); - em.clear(); - Runtime.getRuntime().gc(); - } - } - return count; - } - - private JsonArray convert(final File file) throws IOException { - /** 这里不进行判断,因为格式是严格约定的,出现意外应该先报错停止 */ - final String json = FileUtils.readFileToString(file, DefaultCharset.charset); - final JsonElement jsonElement = pureGsonDateFormated.fromJson(json, JsonElement.class); - final JsonObject jsonObject = jsonElement.getAsJsonObject(); - return jsonObject.get("normals").getAsJsonArray(); - } - - private void clean(final Class cls, final EntityManager em, - final StorageMappings storageMappings) throws Exception { - List list = null; - StorageMapping mapping = null; - do { - if (ListTools.isNotEmpty(list)) { - em.getTransaction().begin(); - for (final T t : list) { - mapping = storageMappings.get(cls, t.getStorage()); - if (null != mapping) { - t.deleteContent(mapping); - } - em.remove(t); - } - em.getTransaction().commit(); - } - final CriteriaBuilder cb = em.getCriteriaBuilder(); - final CriteriaQuery cq = cb.createQuery(cls); - final Root root = cq.from(cls); - cq.select(root); - list = em.createQuery(cq).setMaxResults(Config.dumpRestoreData().getBatchSize()).getResultList(); - } while (ListTools.isNotEmpty(list)); - } - -} \ No newline at end of file diff --git a/o2server/x_file_core_entity/src/main/java/com/x/file/core/entity/personal/Attachment.java b/o2server/x_file_core_entity/src/main/java/com/x/file/core/entity/personal/Attachment.java index abbf1975ac00ea1d98422beca5333c439663c563..33b584e5efce1cc7729b1dac686a34284b0ccccd 100644 --- a/o2server/x_file_core_entity/src/main/java/com/x/file/core/entity/personal/Attachment.java +++ b/o2server/x_file_core_entity/src/main/java/com/x/file/core/entity/personal/Attachment.java @@ -36,7 +36,7 @@ import com.x.base.core.project.annotation.FieldDescribe; import com.x.base.core.project.tools.DateTools; import com.x.file.core.entity.PersistenceProperties; -@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) +@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) @Entity @Table(name = PersistenceProperties.Personal.Attachment.table, uniqueConstraints = { @UniqueConstraint(name = PersistenceProperties.Personal.Attachment.table + JpaObject.IndexNameMiddle diff --git a/o2server/x_meeting_core_entity/src/main/java/com/x/meeting/core/entity/Attachment.java b/o2server/x_meeting_core_entity/src/main/java/com/x/meeting/core/entity/Attachment.java index 1a4a76f26f84c82f6a7bb6c074499a28201f3f6b..586db8a2df91fe746eda5c1cd8a012c5a7fde29d 100644 --- a/o2server/x_meeting_core_entity/src/main/java/com/x/meeting/core/entity/Attachment.java +++ b/o2server/x_meeting_core_entity/src/main/java/com/x/meeting/core/entity/Attachment.java @@ -23,7 +23,7 @@ import com.x.base.core.entity.annotation.CheckPersist; import com.x.base.core.entity.annotation.ContainerEntity; import com.x.base.core.project.annotation.FieldDescribe; -@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) +@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) @Entity @Table(name = PersistenceProperties.Attachment.table, uniqueConstraints = { @UniqueConstraint(name = PersistenceProperties.Attachment.table + JpaObject.IndexNameMiddle diff --git a/o2server/x_message_core_entity/src/main/java/com/x/message/core/entity/IMMsgFile.java b/o2server/x_message_core_entity/src/main/java/com/x/message/core/entity/IMMsgFile.java index 90c6f86b0d5f57577d2c87bb6f4fdfdab6167670..745b16524bcd1dcff918f2c725bb751bda8fc090 100644 --- a/o2server/x_message_core_entity/src/main/java/com/x/message/core/entity/IMMsgFile.java +++ b/o2server/x_message_core_entity/src/main/java/com/x/message/core/entity/IMMsgFile.java @@ -20,7 +20,7 @@ import java.util.Date; */ -@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) +@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) @Entity @Table(name = PersistenceProperties.IMMsgFile.table, uniqueConstraints = { @UniqueConstraint(name = PersistenceProperties.IMMsgFile.table + JpaObject.IndexNameMiddle diff --git a/o2server/x_okr_core_entity/src/main/java/com/x/okr/entity/OkrAttachmentFileInfo.java b/o2server/x_okr_core_entity/src/main/java/com/x/okr/entity/OkrAttachmentFileInfo.java index 3323bd638c3352a2a2e3c0e8224acfbcaa3dbd30..61d33a2c223844566372c8b077cd61f1cf2bb77f 100644 --- a/o2server/x_okr_core_entity/src/main/java/com/x/okr/entity/OkrAttachmentFileInfo.java +++ b/o2server/x_okr_core_entity/src/main/java/com/x/okr/entity/OkrAttachmentFileInfo.java @@ -30,7 +30,7 @@ import com.x.base.core.project.tools.DateTools; * 附件文件信息管理表 * */ -@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) +@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) @Entity @Table(name = PersistenceProperties.OkrAttachmentFileInfo.table, uniqueConstraints = { @UniqueConstraint(name = PersistenceProperties.OkrAttachmentFileInfo.table + JpaObject.IndexNameMiddle diff --git a/o2server/x_processplatform_core_entity/src/main/java/com/x/processplatform/core/entity/content/Attachment.java b/o2server/x_processplatform_core_entity/src/main/java/com/x/processplatform/core/entity/content/Attachment.java index 30a8fc5a7d5c8dd67427540ba6393cdd158267cb..1ec45c5cc8d75efa3347a21eb9fc940271d3f8a7 100644 --- a/o2server/x_processplatform_core_entity/src/main/java/com/x/processplatform/core/entity/content/Attachment.java +++ b/o2server/x_processplatform_core_entity/src/main/java/com/x/processplatform/core/entity/content/Attachment.java @@ -38,7 +38,7 @@ import com.x.base.core.project.tools.DateTools; import com.x.processplatform.core.entity.PersistenceProperties; import com.x.processplatform.core.entity.element.ActivityType; -@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) +@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) @Entity @Table(name = PersistenceProperties.Content.Attachment.table, uniqueConstraints = { @UniqueConstraint(name = PersistenceProperties.Content.Attachment.table + JpaObject.IndexNameMiddle diff --git a/o2server/x_program_center_core_entity/src/main/java/com/x/program/center/core/entity/Structure.java b/o2server/x_program_center_core_entity/src/main/java/com/x/program/center/core/entity/Structure.java index 7d14cf4610108bb1f364613567037a7c53feb227..1a27fdb178b9109aaf86de481087a0b444340675 100644 --- a/o2server/x_program_center_core_entity/src/main/java/com/x/program/center/core/entity/Structure.java +++ b/o2server/x_program_center_core_entity/src/main/java/com/x/program/center/core/entity/Structure.java @@ -25,7 +25,7 @@ import com.x.base.core.project.annotation.FieldDescribe; import com.x.base.core.project.tools.StringTools; @Entity -@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) +@ContainerEntity(dumpSize = 10, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong) @Table(name = PersistenceProperties.Structure.table, uniqueConstraints = { @UniqueConstraint(name = PersistenceProperties.Structure.table + JpaObject.IndexNameMiddle + JpaObject.DefaultUniqueConstraintSuffix, columnNames = { JpaObject.IDCOLUMN,