Skip to content

o2oa
o2oa

ghsby / o2oa
落后 Fork 源项目 2880 个版本

Fork自 浙江兰德纵横网络技术股份有限公司 / o2oa

通知 1
Star 1
Fork 0
o2oa
o2oa

体验新版 GitCode，发现更多精彩内容 >>

提交 e753e163 编写于 作者: Z zhourui
浏览文件
操作

删除dump storage 和 restore storage

上级 dcdbec91
隐藏空白更改
内联 并排
Showing with 712 addition and 1024 deletion
o2server/configSample/dumpRestoreData.json
浏览文件 @ e753e163
......@@ -2,11 +2,15 @@
"enable": false,
"includes": [],
"excludes": [],
"batchSize": 1000.0,
"mode": "lite",
"parallel": true,
"redistribute": true,
"exceptionInvalidStorage": true,
"###enable": "是否启用.###",
"###includes": "导出导入包含对象,可以使用通配符*.###",
"###excludes": "导出导入排除对象,可以使用通配符*.###",
"###batchSize": "批量对象数量.###",
"###mode": "导出数据模式,lite|full,默认使用lite###"
"###mode": "导出数据模式,lite|full,默认使用lite###",
"###parallel": "使用并行导出,默认true###",
"###redistribute": "是否进行重新分布.###",
"###exceptionInvalidStorage": "无法获取storage是否升起错误.###"
}
\ No newline at end of file
o2server/configSample/node_127.0.0.1.json
浏览文件 @ e753e163
......@@ -138,16 +138,6 @@
"###size": "最大保留份数,超过将自动删除最久的数据.###",
"###path": "备份路径###"
},
"dumpStorage": {
"enable": true,
"cron": "",
"size": 7.0,
"path": "",
"###enable": "是否启用,默认每天凌晨4点进行备份.###",
"###cron": "定时任务cron表达式###",
"###size": "最大保留份数,超过将自动删除最久的数据.###",
"###path": "备份路径###"
},
"restoreData": {
"enable": false,
"cron": "",
......@@ -156,14 +146,6 @@
"###cron": "定时任务cron表达式###",
"###path": "恢复路径###"
},
"restoreStorage": {
"enable": false,
"cron": "",
"path": "",
"###enable": "是否启用.###",
"###cron": "定时任务cron表达式###",
"###path": "恢复路径###"
},
"nodeAgentEnable": true,
"nodeAgentPort": 20010.0,
"nodeAgentEncrypt": true,
......@@ -178,9 +160,7 @@
"###storage": "Storage服务器配置###",
"###logLevel": "日志级别,默认当前节点的slf4j日志级别,通过系统变量\"org.slf4j.simpleLogger.defaultLogLevel\"设置到当前jvm中.###",
"###dumpData": "定时数据导出配置###",
"###dumpStorage": "定时存储文件导出配置###",
"###restoreData": "定时数据导入配置###",
"###restoreStorage": "定时存储文件导入配置###",
"###logSize": "日志文件保留天数.###",
"###auditLogSize": "审计日志文件保留天数.###",
"###nodeAgentEnable": "是否启用节点代理###",
......
o2server/pom.xml
浏览文件 @ e753e163
......@@ -494,7 +494,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
<version>2.7</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
......
o2server/x_base_core_project/src/main/java/com/x/base/core/project/build/CheckCore.java
浏览文件 @ e753e163
......@@ -19,6 +19,8 @@ import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.openjpa.persistence.jdbc.ContainerTable;
import com.x.base.core.entity.JpaObject;
import com.x.base.core.entity.JsonProperties;
import com.x.base.core.entity.StorageObject;
import com.x.base.core.entity.annotation.ContainerEntity;
import com.x.base.core.project.annotation.FieldDescribe;
import com.x.base.core.project.annotation.Module;
......@@ -44,6 +46,7 @@ public class CheckCore {
checkTableNameUniqueConstraintName(classes);
checkIdCreateTimeUpdateTimeSequenceIndex(classes);
checkEnum(classes);
checkDumpSize(classes);
}
}
......@@ -215,17 +218,33 @@ public class CheckCore {
for (Class<?> cls : classes) {
List<Field> fields = FieldUtils.getFieldsListWithAnnotation(cls, Column.class);
for (Field field : fields) {
if ((!String.class.isAssignableFrom(field.getType())) && (!field.getType().isEnum())) {
Column column = field.getAnnotation(Column.class);
if (column.length() != 255) {
System.err.println(String.format("checkColumnLength error: class: %s, field: %s.",
cls.getName(), field.getName()));
if ((!String.class.isAssignableFrom(field.getType())) && (!field.getType().isEnum())
&& (!JsonProperties.class.isAssignableFrom(field.getType()))) {
Lob lob = cls.getAnnotation(Lob.class);
if (null != lob) {
Column column = field.getAnnotation(Column.class);
if (column.length() > 255) {
System.err.println(String.format("checkColumnLength error: class: %s, field: %s.",
cls.getName(), field.getName()));
}
}
}
}
}
}
/* 检查StorageObject的dumpSize是否设置正确 */
public static void checkDumpSize(List<Class<?>> classes) throws Exception {
for (Class<?> cls : classes) {
if (StorageObject.class.isAssignableFrom(cls)) {
ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
if (containerEntity.dumpSize() > 10) {
System.err.println(String.format("checkDumpSize error: class: %s.", cls.getName()));
}
}
}
}
public static void checkIdUnique(List<Class<?>> classes) throws Exception {
for (Class<?> cls : classes) {
Field idField = FieldUtils.getField(cls, JpaObject.id_FIELDNAME, true);
......
o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java
浏览文件 @ e753e163
......@@ -11,22 +11,27 @@ import com.x.base.core.project.tools.ListTools;
public class DumpRestoreData extends ConfigObject {
public static String TYPE_FULL = "full";
public static String TYPE_LITE = "lite";
private static final long serialVersionUID = 8910820385137391619L;
public static DumpRestoreData defaultInstance() {
return new DumpRestoreData();
}
public static final int default_batchSize = 1000;
public static final String default_type = TYPE_LITE;
public static final String TYPE_FULL = "full";
public static final String TYPE_LITE = "lite";
public static final String DEFAULT_TYPE = TYPE_LITE;
public static final Boolean DEFAULT_PARALLEL = true;
public static final Boolean DEFAULT_REDISTRIBUTE = true;
public static final Boolean DEFAULT_EXCEPTIONINVALIDSTORAGE = true;
public DumpRestoreData() {
this.enable = false;
this.includes = new ArrayList<String>();
this.excludes = new ArrayList<String>();
this.batchSize = default_batchSize;
this.mode = default_type;
this.includes = new ArrayList<>();
this.excludes = new ArrayList<>();
this.mode = DEFAULT_TYPE;
this.parallel = DEFAULT_PARALLEL;
this.redistribute = DEFAULT_REDISTRIBUTE;
this.exceptionInvalidStorage = DEFAULT_EXCEPTIONINVALIDSTORAGE;
}
@FieldDescribe("是否启用.")
......@@ -38,12 +43,30 @@ public class DumpRestoreData extends ConfigObject {
@FieldDescribe("导出导入排除对象,可以使用通配符*.")
private List<String> excludes;
@FieldDescribe("批量对象数量.")
private Integer batchSize;
@FieldDescribe("导出数据模式,lite|full,默认使用lite")
private String mode;
@FieldDescribe("使用并行导出,默认true")
private Boolean parallel;
@FieldDescribe("是否进行重新分布.")
private Boolean redistribute;
@FieldDescribe("无法获取storage是否升起错误.")
private Boolean exceptionInvalidStorage;
public Boolean getRedistribute() {
return BooleanUtils.isNotFalse(redistribute);
}
public Boolean getExceptionInvalidStorage() {
return BooleanUtils.isNotFalse(exceptionInvalidStorage);
}
public Boolean getParallel() {
return BooleanUtils.isNotFalse(parallel);
}
public String getMode() {
return StringUtils.equals(TYPE_FULL, mode) ? TYPE_FULL : TYPE_LITE;
}
......@@ -68,13 +91,6 @@ public class DumpRestoreData extends ConfigObject {
return list;
}
public Integer getBatchSize() {
if ((null == this.batchSize) || (this.batchSize < 1)) {
return default_batchSize;
}
return this.batchSize;
}
public void setIncludes(List<String> includes) {
this.includes = includes;
}
......@@ -83,8 +99,16 @@ public class DumpRestoreData extends ConfigObject {
this.excludes = excludes;
}
public void setBatchSize(Integer batchSize) {
this.batchSize = batchSize;
public void setParallel(Boolean parallel) {
this.parallel = parallel;
}
public void setRedistribute(Boolean redistribute) {
this.redistribute = redistribute;
}
public void setExceptionInvalidStorage(Boolean exceptionInvalidStorage) {
this.exceptionInvalidStorage = exceptionInvalidStorage;
}
public void setEnable(Boolean enable) {
......
o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/Node.java
浏览文件 @ e753e163
package com.x.base.core.project.config;
import com.x.base.core.project.annotation.FieldDescribe;
import com.x.base.core.project.tools.DateTools;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import com.x.base.core.project.annotation.FieldDescribe;
import com.x.base.core.project.tools.DateTools;
public class Node extends ConfigObject {
public static final Integer default_nodeAgentPort = 20010;
public static final String default_banner = "O2OA";
public static final Integer default_logSize = 14;
public static final Integer DEFAULT_NODEAGENTPORT = 20010;
public static final String DEFAULT_BANNER = "O2OA";
public static final Integer DEFAULT_LOGSIZE = 14;
public static Node defaultInstance() {
Node o = new Node();
......@@ -23,12 +23,10 @@ public class Node extends ConfigObject {
o.storage = StorageServer.defaultInstance();
o.logLevel = "warn";
o.dumpData = new ScheduleDumpData();
o.dumpStorage = new ScheduleDumpStorage();
o.restoreData = new ScheduleRestoreData();
o.restoreStorage = new ScheduleRestoreStorage();
o.nodeAgentEnable = true;
o.nodeAgentEncrypt = true;
o.nodeAgentPort = default_nodeAgentPort;
o.nodeAgentPort = DEFAULT_NODEAGENTPORT;
o.quickStartWebApp = false;
o.autoStart = true;
return o;
......@@ -52,12 +50,8 @@ public class Node extends ConfigObject {
private String logLevel;
@FieldDescribe("定时数据导出配置")
private ScheduleDumpData dumpData;
@FieldDescribe("定时存储文件导出配置")
private ScheduleDumpStorage dumpStorage;
@FieldDescribe("定时数据导入配置")
private ScheduleRestoreData restoreData;
@FieldDescribe("定时存储文件导入配置")
private ScheduleRestoreStorage restoreStorage;
@FieldDescribe("日志文件保留天数.")
private Integer logSize;
@FieldDescribe("审计日志文件保留天数.")
......@@ -83,7 +77,6 @@ public class Node extends ConfigObject {
}
/* 20191009兼容centerServer end */
public Boolean getEraseContentEnable() {
return BooleanUtils.isNotFalse(eraseContentEnable);
}
......@@ -136,14 +129,14 @@ public class Node extends ConfigObject {
}
public String getBanner() {
return StringUtils.isBlank(this.banner) ? default_banner : this.banner;
return StringUtils.isBlank(this.banner) ? DEFAULT_BANNER : this.banner;
}
public Integer logSize() {
if ((this.logSize != null) && (this.logSize > 0)) {
return this.logSize;
}
return default_logSize;
return DEFAULT_LOGSIZE;
}
public Boolean getQuickStartWebApp() {
......@@ -152,7 +145,7 @@ public class Node extends ConfigObject {
public Integer nodeAgentPort() {
if (null == this.nodeAgentPort || this.nodeAgentPort < 0) {
return default_nodeAgentPort;
return DEFAULT_NODEAGENTPORT;
}
return this.nodeAgentPort;
}
......@@ -173,18 +166,10 @@ public class Node extends ConfigObject {
return (dumpData == null) ? new ScheduleDumpData() : this.dumpData;
}
public ScheduleDumpStorage dumpStorage() {
return (dumpStorage == null) ? new ScheduleDumpStorage() : this.dumpStorage;
}
public ScheduleRestoreData restoreData() {
return (restoreData == null) ? new ScheduleRestoreData() : this.restoreData;
}
public ScheduleRestoreStorage restoreStorage() {
return (restoreStorage == null) ? new ScheduleRestoreStorage() : this.restoreStorage;
}
public static class ScheduleDumpData extends ConfigObject {
public static ScheduleDumpData defaultInstance() {
......@@ -225,46 +210,6 @@ public class Node extends ConfigObject {
}
public static class ScheduleDumpStorage extends ConfigObject {
public static ScheduleDumpStorage defaultInstance() {
return new ScheduleDumpStorage();
}
public boolean available() {
return DateTools.cronAvailable(this.cron());
}
@FieldDescribe("是否启用,默认每天凌晨4点进行备份.")
private Boolean enable = true;
@FieldDescribe("定时任务cron表达式")
private String cron = "";
@FieldDescribe("最大保留份数,超过将自动删除最久的数据.")
private Integer size = 7;
@FieldDescribe("备份路径")
private String path = "";
public Boolean enable() {
return (BooleanUtils.isTrue(this.enable)) ? true : false;
}
public String cron() {
return (null == cron) ? "5 0 4 * * ?" : this.cron;
}
public Integer size() {
return (null == size) ? 14 : this.size;
}
public String path() {
return StringUtils.trim(path);
}
}
public static class ScheduleRestoreData extends ConfigObject {
public static ScheduleRestoreData defaultInstance() {
......@@ -298,37 +243,4 @@ public class Node extends ConfigObject {
}
public static class ScheduleRestoreStorage extends ConfigObject {
public static ScheduleRestoreStorage defaultInstance() {
return new ScheduleRestoreStorage();
}
public boolean available() {
return DateTools.cronAvailable(this.cron) && StringUtils.isNotEmpty(this.path);
}
@FieldDescribe("是否启用.")
private Boolean enable = false;
@FieldDescribe("定时任务cron表达式")
private String cron = "";
@FieldDescribe("恢复路径")
private String path = "";
public Boolean enable() {
return (BooleanUtils.isTrue(this.enable)) ? true : false;
}
public String cron() {
return (null == cron) ? "" : this.cron;
}
public String path() {
return StringUtils.trim(path);
}
}
}
o2server/x_base_core_project/src/main/java/com/x/base/core/project/tools/StringTools.java
浏览文件 @ e753e163
......@@ -35,7 +35,26 @@ public class StringTools {
/** 中文,英文,数字,-,.· 【】（） */
public static final Pattern SIMPLY_REGEX = Pattern
.compile("^[\u4e00-\u9fa5a-zA-Z0-9\\_\\(\\)\\-\\ \\.\\ \\·\\【\\】\\（\\）]*$");
public static final Pattern FILENAME_REGEX = Pattern.compile("[^/\\\\<>*?|\"]+(\\.?)[^/\\\\<>*?|\"]+");
/**
* MSDN
* https://docs.microsoft.com/zh-cn/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#file_and_directory_names
*/
public static final Pattern FILENAME_REGEX = Pattern.compile(
"# Match a valid Windows filename (unspecified file system). \n"
+ "^ # Anchor to start of string. \n"
+ "(?! # Assert filename is not: CON, PRN, \n"
+ " (?: # AUX, NUL, COM1, COM2, COM3, COM4, \n"
+ " CON|PRN|AUX|NUL| # COM5, COM6, COM7, COM8, COM9, \n"
+ " COM[1-9]|LPT[1-9] # LPT1, LPT2, LPT3, LPT4, LPT5, \n"
+ " ) # LPT6, LPT7, LPT8, and LPT9... \n"
+ " (?:\\.[^.]*)? # followed by optional extension \n"
+ " $ # and end of string \n"
+ ") # End negative lookahead assertion. \n"
+ "[^<>:\"/\\\\|?*\\x00-\\x1F]* # Zero or more valid filename chars.\n"
+ "[^<>:\"/\\\\|?*\\x00-\\x1F\\ .] # Last char is not a space or dot. \n"
+ "$ # Anchor to end of string. ",
Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.COMMENTS);
/**
* RFC822 compliant regex adapted for Java
* http://stackoverflow.com/questions/8204680/java-regex-email
......@@ -480,37 +499,37 @@ public class StringTools {
while (tok.hasMoreTokens()) {
String nextTok = tok.nextToken();
switch (state) {
case inQuote:
if ("\'".equals(nextTok)) {
lastTokenHasBeenQuoted = true;
state = normal;
} else {
current.append(nextTok);
}
break;
case inDoubleQuote:
if ("\"".equals(nextTok)) {
lastTokenHasBeenQuoted = true;
state = normal;
} else {
current.append(nextTok);
}
break;
default:
if ("\'".equals(nextTok)) {
state = inQuote;
} else if ("\"".equals(nextTok)) {
state = inDoubleQuote;
} else if (" ".equals(nextTok)) {
if (lastTokenHasBeenQuoted || current.length() > 0) {
result.add(current.toString());
current.setLength(0);
}
} else {
current.append(nextTok);
case inQuote:
if ("\'".equals(nextTok)) {
lastTokenHasBeenQuoted = true;
state = normal;
} else {
current.append(nextTok);
}
break;
case inDoubleQuote:
if ("\"".equals(nextTok)) {
lastTokenHasBeenQuoted = true;
state = normal;
} else {
current.append(nextTok);
}
break;
default:
if ("\'".equals(nextTok)) {
state = inQuote;
} else if ("\"".equals(nextTok)) {
state = inDoubleQuote;
} else if (" ".equals(nextTok)) {
if (lastTokenHasBeenQuoted || current.length() > 0) {
result.add(current.toString());
current.setLength(0);
}
lastTokenHasBeenQuoted = false;
break;
} else {
current.append(nextTok);
}
lastTokenHasBeenQuoted = false;
break;
}
}
if (lastTokenHasBeenQuoted || current.length() > 0) {
......
o2server/x_bbs_core_entity/src/main/java/com/x/bbs/entity/BBSSubjectAttachment.java
浏览文件 @ e753e163
......@@ -29,7 +29,7 @@ import com.x.base.core.project.tools.DateTools;
/**
* 附件信息管理表
*/
@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@ContainerEntity(dumpSize = 10, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@Entity
@Table(name = PersistenceProperties.BBSSubjectAttachment.table, uniqueConstraints = {
@UniqueConstraint(name = PersistenceProperties.BBSSubjectAttachment.table + JpaObject.IndexNameMiddle
......
o2server/x_cms_core_entity/src/main/java/com/x/cms/core/entity/FileInfo.java
浏览文件 @ e753e163
......@@ -39,7 +39,7 @@ import com.x.base.core.project.tools.DateTools;
* 内容管理应用目录分类信息
*
*/
@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@Entity
@Table(name = PersistenceProperties.FileInfo.table, uniqueConstraints = {
@UniqueConstraint(name = PersistenceProperties.FileInfo.table + JpaObject.IndexNameMiddle
......
o2server/x_console/src/main/java/com/x/server/console/DumpStorageTask.java
浏览文件 @ e753e163
package com.x.server.console;
import java.io.File;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.FalseFileFilter;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import com.x.base.core.project.config.Config;
import com.x.base.core.project.logger.Logger;
import com.x.base.core.project.logger.LoggerFactory;
import com.x.server.console.action.DumpStorage;
public class DumpStorageTask implements Job {
private static Logger logger = LoggerFactory.getLogger(DumpStorageTask.class);
@Override
public void execute(JobExecutionContext arg0) throws JobExecutionException {
try {
logger.print("schedule dump storage task start.");
DumpStorage action = new DumpStorage();
action.execute(Config.currentNode().dumpStorage().path());
if (Config.currentNode().dumpStorage().size() > 0) {
File dir = new File(Config.base(), "local/dump");
List<File> list = new ArrayList<>();
if (dir.exists() && dir.isDirectory()) {
for (File f : FileUtils.listFilesAndDirs(dir, FalseFileFilter.FALSE, new RegexFileFilter(
"^dumpStorage_[1,2][0,9][0-9][0-9][0,1][0-9][0-3][0-9][0-5][0-9][0-5][0-9][0-5][0-9]$"))) {
if (dir != f) {
list.add(f);
}
}
list = list.stream().sorted(Comparator.comparing(File::getName).reversed())
.collect(Collectors.toList());
if (list.size() > Config.currentNode().dumpStorage().size()) {
for (int i = Config.currentNode().dumpStorage().size(); i < list.size(); i++) {
File file = list.get(i);
logger.print("dumpStorageTask delete{}.", file.getAbsolutePath());
FileUtils.forceDelete(file);
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
\ No newline at end of file
//package com.x.server.console;
//
//import java.io.File;
//import java.util.ArrayList;
//import java.util.Comparator;
//import java.util.List;
//import java.util.stream.Collectors;
//
//import org.apache.commons.io.FileUtils;
//import org.apache.commons.io.filefilter.FalseFileFilter;
//import org.apache.commons.io.filefilter.RegexFileFilter;
//import org.quartz.Job;
//import org.quartz.JobExecutionContext;
//import org.quartz.JobExecutionException;
//
//import com.x.base.core.project.config.Config;
//import com.x.base.core.project.logger.Logger;
//import com.x.base.core.project.logger.LoggerFactory;
//import com.x.server.console.action.DumpStorage;
//
//public class DumpStorageTask implements Job {
//
// private static Logger logger = LoggerFactory.getLogger(DumpStorageTask.class);
//
// @Override
// public void execute(JobExecutionContext arg0) throws JobExecutionException {
// try {
// logger.print("schedule dump storage task start.");
// DumpStorage action = new DumpStorage();
// action.execute(Config.currentNode().dumpStorage().path());
// if (Config.currentNode().dumpStorage().size() > 0) {
// File dir = new File(Config.base(), "local/dump");
// List<File> list = new ArrayList<>();
// if (dir.exists() && dir.isDirectory()) {
// for (File f : FileUtils.listFilesAndDirs(dir, FalseFileFilter.FALSE, new RegexFileFilter(
// "^dumpStorage_[1,2][0,9][0-9][0-9][0,1][0-9][0-3][0-9][0-5][0-9][0-5][0-9][0-5][0-9]$"))) {
// if (dir != f) {
// list.add(f);
// }
// }
// list = list.stream().sorted(Comparator.comparing(File::getName).reversed())
// .collect(Collectors.toList());
// if (list.size() > Config.currentNode().dumpStorage().size()) {
// for (int i = Config.currentNode().dumpStorage().size(); i < list.size(); i++) {
// File file = list.get(i);
// logger.print("dumpStorageTask delete{}.", file.getAbsolutePath());
// FileUtils.forceDelete(file);
// }
// }
// }
// }
// } catch (Exception e) {
// e.printStackTrace();
// }
// }
//}
\ No newline at end of file
o2server/x_console/src/main/java/com/x/server/console/Main.java
浏览文件 @ e753e163
......@@ -3,8 +3,11 @@ package com.x.server.console;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
......@@ -12,12 +15,22 @@ import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.channels.FileLock;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Matcher;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.quartz.Scheduler;
import com.x.base.core.project.config.ApplicationServer;
import com.x.base.core.project.config.CenterServer;
import com.x.base.core.project.config.Config;
......@@ -34,14 +47,6 @@ import com.x.server.console.action.ActionVersion;
import com.x.server.console.log.LogTools;
import com.x.server.console.server.Servers;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jetty.deploy.App;
import org.eclipse.jetty.deploy.DeploymentManager;
import org.quartz.Scheduler;
public class Main {
private static final String MANIFEST_FILENAME = "manifest.cfg";
......@@ -50,6 +55,7 @@ public class Main {
public static void main(String[] args) throws Exception {
String base = getBasePath();
pid(base);
scanWar(base);
loadJars(base);
/* getVersion需要FileUtils在后面运行 */
......@@ -620,4 +626,12 @@ public class Main {
return false;
}
private static void pid(String base) throws IOException {
RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
String jvmName = runtimeBean.getName();
long pid = Long.parseLong(jvmName.split("@")[0]);
Path path = Paths.get(base, "pid.log");
Files.write(path, Long.toString(pid).getBytes(), StandardOpenOption.CREATE,
StandardOpenOption.TRUNCATE_EXISTING);
}
}
\ No newline at end of file
o2server/x_console/src/main/java/com/x/server/console/RestoreStorageTask.java
浏览文件 @ e753e163
package com.x.server.console;
import com.x.base.core.project.config.Config;
import com.x.base.core.project.logger.Logger;
import com.x.base.core.project.logger.LoggerFactory;
import com.x.server.console.action.RestoreStorage;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
public class RestoreStorageTask implements Job {
private static Logger logger = LoggerFactory.getLogger(RestoreStorageTask.class);
@Override
public void execute(JobExecutionContext arg0) throws JobExecutionException {
try {
logger.print("schedule restore storage task start, restore from:{}.",
Config.currentNode().restoreData().path());
RestoreStorage action = new RestoreStorage();
action.execute(Config.currentNode().restoreStorage().path());
} catch (Exception e) {
throw new JobExecutionException(e);
}
}
}
\ No newline at end of file
//package com.x.server.console;
//
//import com.x.base.core.project.config.Config;
//import com.x.base.core.project.logger.Logger;
//import com.x.base.core.project.logger.LoggerFactory;
//import com.x.server.console.action.RestoreStorage;
//
//import org.quartz.Job;
//import org.quartz.JobExecutionContext;
//import org.quartz.JobExecutionException;
//
//public class RestoreStorageTask implements Job {
//
// private static Logger logger = LoggerFactory.getLogger(RestoreStorageTask.class);
//
// @Override
// public void execute(JobExecutionContext arg0) throws JobExecutionException {
// try {
// logger.print("schedule restore storage task start, restore from:{}.",
// Config.currentNode().restoreData().path());
// RestoreStorage action = new RestoreStorage();
// action.execute(Config.currentNode().restoreStorage().path());
// } catch (Exception e) {
// throw new JobExecutionException(e);
// }
//
// }
//
//}
\ No newline at end of file
o2server/x_console/src/main/java/com/x/server/console/SchedulerBuilder.java
浏览文件 @ e753e163
......@@ -28,14 +28,14 @@ public class SchedulerBuilder {
.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpData().cron())).build();
scheduler.scheduleJob(jobDetail, trigger);
}
if (Config.currentNode().dumpStorage().enable() && Config.currentNode().dumpStorage().available()) {
JobDetail jobDetail = JobBuilder.newJob(DumpStorageTask.class)
.withIdentity(DumpStorageTask.class.getName(), scheduleGroup).withDescription(Config.node())
.build();
Trigger trigger = TriggerBuilder.newTrigger().withIdentity(DumpStorageTask.class.getName(), scheduleGroup)
.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpStorage().cron())).build();
scheduler.scheduleJob(jobDetail, trigger);
}
// if (Config.currentNode().dumpStorage().enable() && Config.currentNode().dumpStorage().available()) {
// JobDetail jobDetail = JobBuilder.newJob(DumpStorageTask.class)
// .withIdentity(DumpStorageTask.class.getName(), scheduleGroup).withDescription(Config.node())
// .build();
// Trigger trigger = TriggerBuilder.newTrigger().withIdentity(DumpStorageTask.class.getName(), scheduleGroup)
// .withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().dumpStorage().cron())).build();
// scheduler.scheduleJob(jobDetail, trigger);
// }
if (Config.currentNode().restoreData().enable() && Config.currentNode().restoreData().available()) {
JobDetail jobDetail = JobBuilder.newJob(RestoreDataTask.class)
.withIdentity(RestoreDataTask.class.getName(), scheduleGroup).withDescription(Config.node())
......@@ -44,16 +44,16 @@ public class SchedulerBuilder {
.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreData().cron())).build();
scheduler.scheduleJob(jobDetail, trigger);
}
if (Config.currentNode().restoreStorage().enable() && Config.currentNode().restoreStorage().available()) {
JobDetail jobDetail = JobBuilder.newJob(RestoreStorageTask.class)
.withIdentity(RestoreStorageTask.class.getName(), scheduleGroup).withDescription(Config.node())
.build();
Trigger trigger = TriggerBuilder.newTrigger()
.withIdentity(RestoreStorageTask.class.getName(), scheduleGroup)
.withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreStorage().cron()))
.build();
scheduler.scheduleJob(jobDetail, trigger);
}
// if (Config.currentNode().restoreStorage().enable() && Config.currentNode().restoreStorage().available()) {
// JobDetail jobDetail = JobBuilder.newJob(RestoreStorageTask.class)
// .withIdentity(RestoreStorageTask.class.getName(), scheduleGroup).withDescription(Config.node())
// .build();
// Trigger trigger = TriggerBuilder.newTrigger()
// .withIdentity(RestoreStorageTask.class.getName(), scheduleGroup)
// .withSchedule(CronScheduleBuilder.cronSchedule(Config.currentNode().restoreStorage().cron()))
// .build();
// scheduler.scheduleJob(jobDetail, trigger);
// }
this.registApplicationsAndVoteCenterTask(scheduler, scheduleGroup);
return scheduler;
......
o2server/x_console/src/main/java/com/x/server/console/action/ActionControl.java
浏览文件 @ e753e163
......@@ -33,9 +33,9 @@ public class ActionControl extends ActionBase {
private static final String CMD_TD = "td";
private static final String CMD_EC = "ec";
private static final String CMD_DD = "dd";
private static final String CMD_DS = "ds";
// private static final String CMD_DS = "ds";
private static final String CMD_RD = "rd";
private static final String CMD_RS = "rs";
// private static final String CMD_RS = "rs";
private static final String CMD_CLH2 = "clh2";
private static final String CMD_UF = "uf";
private static final String CMD_DDL = "ddl";
......@@ -63,12 +63,12 @@ public class ActionControl extends ActionBase {
ec(cmd);
} else if (cmd.hasOption(CMD_DD)) {
dd(cmd);
} else if (cmd.hasOption(CMD_DS)) {
ds(cmd);
// } else if (cmd.hasOption(CMD_DS)) {
// ds(cmd);
} else if (cmd.hasOption(CMD_RD)) {
rd(cmd);
} else if (cmd.hasOption(CMD_RS)) {
rs(cmd);
// } else if (cmd.hasOption(CMD_RS)) {
// rs(cmd);
} else if (cmd.hasOption(CMD_CLH2)) {
clh2(cmd);
} else if (cmd.hasOption(CMD_UF)) {
......@@ -96,9 +96,9 @@ public class ActionControl extends ActionBase {
options.addOption(tdOption());
options.addOption(ecOption());
options.addOption(ddOption());
options.addOption(dsOption());
// options.addOption(dsOption());
options.addOption(rdOption());
options.addOption(rsOption());
// options.addOption(rsOption());
options.addOption(clh2Option());
options.addOption(ufOption());
options.addOption(ddlOption());
......@@ -145,20 +145,20 @@ public class ActionControl extends ActionBase {
.desc("导出数据库服务器的数据转换成json格式保存到本地文件.").build();
}
private static Option dsOption() {
return Option.builder("ds").longOpt("dumpStorage").argName("path").hasArg().optionalArg(true)
.desc("导出存储服务器的文件数据转换成json格式保存到本地文件.").build();
}
// private static Option dsOption() {
// return Option.builder("ds").longOpt("dumpStorage").argName("path").hasArg().optionalArg(true)
// .desc("导出存储服务器的文件数据转换成json格式保存到本地文件.").build();
// }
private static Option rdOption() {
return Option.builder("rd").longOpt("restoreData").argName("path or date").hasArg()
.desc("将导出的json格式数据恢复到数据库服务器.").build();
}
private static Option rsOption() {
return Option.builder("rs").longOpt("restoreStorage").argName("path or date").hasArg()
.desc("将导出的json格式文件数据恢复到存储服务器.").build();
}
// private static Option rsOption() {
// return Option.builder("rs").longOpt("restoreStorage").argName("path or date").hasArg()
// .desc("将导出的json格式文件数据恢复到存储服务器.").build();
// }
private static Option ufOption() {
return Option.builder("uf").longOpt("updateFile").argName("path").hasArg().desc("升级服务器,升级前请注意备份.").build();
......@@ -229,11 +229,11 @@ public class ActionControl extends ActionBase {
dumpData.execute(path);
}
private void ds(CommandLine cmd) throws Exception {
String path = Objects.toString(cmd.getOptionValue(CMD_DS), "");
DumpStorage dumpStorage = new DumpStorage();
dumpStorage.execute(path);
}
// private void ds(CommandLine cmd) throws Exception {
// String path = Objects.toString(cmd.getOptionValue(CMD_DS), "");
// DumpStorage dumpStorage = new DumpStorage();
// dumpStorage.execute(path);
// }
private void rd(CommandLine cmd) throws Exception {
String path = Objects.toString(cmd.getOptionValue(CMD_RD), "");
......@@ -241,11 +241,11 @@ public class ActionControl extends ActionBase {
restoreData.execute(path);
}
private void rs(CommandLine cmd) throws Exception {
String path = Objects.toString(cmd.getOptionValue(CMD_RS), "");
RestoreStorage restoreStorage = new RestoreStorage();
restoreStorage.execute(path);
}
// private void rs(CommandLine cmd) throws Exception {
// String path = Objects.toString(cmd.getOptionValue(CMD_RS), "");
// RestoreStorage restoreStorage = new RestoreStorage();
// restoreStorage.execute(path);
// }
private void hs(CommandLine cmd) {
final Integer repeat = this.getArgInteger(cmd, CMD_HS, 1);
......
o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java
浏览文件 @ e753e163
package com.x.server.console.action;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
......@@ -16,152 +20,198 @@ import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.file.PathUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.openjpa.persistence.OpenJPAPersistence;
import com.google.gson.Gson;
import com.x.base.core.container.factory.PersistenceXmlHelper;
import com.x.base.core.entity.JpaObject;
import com.x.base.core.entity.StorageObject;
import com.x.base.core.entity.annotation.ContainerEntity;
import com.x.base.core.entity.annotation.ContainerEntity.Reference;
import com.x.base.core.project.config.Config;
import com.x.base.core.project.config.DumpRestoreData;
import com.x.base.core.project.config.StorageMapping;
import com.x.base.core.project.config.StorageMappings;
import com.x.base.core.project.gson.XGsonBuilder;
import com.x.base.core.project.logger.Logger;
import com.x.base.core.project.logger.LoggerFactory;
import com.x.base.core.project.tools.DateTools;
import com.x.base.core.project.tools.DefaultCharset;
import com.x.base.core.project.tools.ListTools;
public class DumpData {
private static Logger logger = LoggerFactory.getLogger(DumpData.class);
private Date start = new Date();
private File dir;
private DumpRestoreDataCatalog catalog;
private Gson pureGsonDateFormated = XGsonBuilder.instance();
@SuppressWarnings("unchecked")
public boolean execute(String path) throws Exception {
Path dir = null;
Date start = new Date();
if (StringUtils.isEmpty(path)) {
this.dir = new File(Config.base(), "local/dump/dumpData_" + DateTools.compact(this.start));
dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + DateTools.compact(start));
} else {
this.dir = new File(path);
if (dir.getAbsolutePath().startsWith(Config.base())) {
logger.print("path can not in base directory.");
dir = Paths.get(path);
if (dir.startsWith(Paths.get(Config.base()))) {
logger.warn("path can not in base directory.");
return false;
}
}
FileUtils.forceMkdir(this.dir);
this.catalog = new DumpRestoreDataCatalog();
/* 初始化完成 */
List<String> containerEntityNames = this.entities();
/** 过滤include exclude 条件 */
List<String> classNames = ListTools.includesExcludesWildcard(containerEntityNames,
Config.dumpRestoreData().getIncludes(), Config.dumpRestoreData().getExcludes());
logger.print("dump data find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml");
PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames);
for (int i = 0; i < classNames.size(); i++) {
Class<JpaObject> cls = (Class<JpaObject>) Class.forName(classNames.get(i));
EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
EntityManager em = emf.createEntityManager();
try {
long estimateCount = this.estimateCount(em, cls);
logger.print("dump data({}/{}): {}, count: {}.", (i + 1), classNames.size(), cls.getName(),
estimateCount);
this.dump(cls, em, estimateCount);
} finally {
em.close();
emf.close();
}
}
FileUtils.write(new File(dir, "catalog.json"), pureGsonDateFormated.toJson(this.catalog),
DefaultCharset.charset);
logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.getAbsolutePath(),
this.count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60);
Files.createDirectories(dir);
Thread thread = new Thread(new RunnableImpl(dir, start));
thread.start();
return true;
}
private <T extends JpaObject> long estimateCount(EntityManager em, Class<T> cls) {
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<Long> cq = cb.createQuery(Long.class);
Root<T> root = cq.from(cls);
cq.select(cb.count(root));
return em.createQuery(cq).getSingleResult();
}
public class RunnableImpl implements Runnable {
private Integer count() {
return this.catalog.values().stream().mapToInt(Integer::intValue).sum();
}
private Path dir;
private Date start;
private DumpRestoreDataCatalog catalog;
private Gson pureGsonDateFormated;
public RunnableImpl(Path dir, Date start) {
this.dir = dir;
this.start = start;
this.catalog = new DumpRestoreDataCatalog();
this.pureGsonDateFormated = XGsonBuilder.instance();
}
private <T> void dump(Class<T> cls, EntityManager em, long total)
throws IOException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
/** 创建最终存储文件的目录 */
File directory = new File(dir, cls.getName());
FileUtils.forceMkdir(directory);
FileUtils.cleanDirectory(directory);
ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
String id = "";
List<T> list = null;
int count = 0;
int loop = 1;
int btach = (int) ((total + 0.0) / containerEntity.dumpSize());
do {
list = this.list(em, cls, id, containerEntity.dumpSize());
if (ListTools.isNotEmpty(list)) {
count = count + list.size();
id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME);
File file = new File(directory, count + ".json");
FileUtils.write(file, pureGsonDateFormated.toJson(list), DefaultCharset.charset);
logger.print("dumping {}/{} part of data:{}.", loop++, btach, cls.getName());
public void run() {
try {
List<String> classNames = entities();
logger.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(),
DateTools.compact(start) + "_dump.xml");
PersistenceXmlHelper.write(xml.toString(), classNames);
StorageMappings storageMappings = Config.storageMappings();
Stream<String> stream = BooleanUtils.isTrue(Config.dumpRestoreData().getParallel())
? classNames.parallelStream()
: classNames.stream();
AtomicInteger idx = new AtomicInteger(1);
stream.forEach(className -> {
String nameOfThread = Thread.currentThread().getName();
EntityManagerFactory emf = null;
EntityManager em = null;
try {
Thread.currentThread().setName(DumpData.class.getName() + ":" + className);
@SuppressWarnings("unchecked")
Class<JpaObject> cls = (Class<JpaObject>) Class.forName(className);
emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(),
PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
em = emf.createEntityManager();
long estimateCount = estimateCount(em, cls);
logger.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(),
cls.getName(), estimateCount);
dump(cls, em, storageMappings, estimateCount);
} catch (Exception e) {
logger.error(new Exception(String.format("dump:%s error.", className), e));
} finally {
Thread.currentThread().setName(nameOfThread);
em.close();
emf.close();
}
});
Files.write(dir.resolve("catalog.json"),
pureGsonDateFormated.toJson(catalog).getBytes(StandardCharsets.UTF_8));
logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(),
count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60);
} catch (Exception e) {
e.printStackTrace();
}
em.clear();
} while (ListTools.isNotEmpty(list));
this.catalog.put(cls.getName(), count);
}
}
@SuppressWarnings("unchecked")
private List<String> entities() throws Exception {
List<String> list = new ArrayList<>();
if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) {
list.addAll((List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES));
return list;
}
for (String str : (List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)) {
Class<?> cls = Class.forName(str);
ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
if (Objects.equals(containerEntity.reference(), Reference.strong)) {
list.add(str);
}
}
return ListTools.includesExcludesWildcard(list, Config.dumpRestoreData().getIncludes(),
Config.dumpRestoreData().getExcludes());
}
private <T> List<T> list(EntityManager em, Class<T> cls, String id, Integer size) {
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<T> cq = cb.createQuery(cls);
Root<T> root = cq.from(cls);
Predicate p = cb.conjunction();
if (StringUtils.isNotEmpty(id)) {
p = cb.greaterThan(root.get(JpaObject.id_FIELDNAME), id);
private <T extends JpaObject> long estimateCount(EntityManager em, Class<T> cls) {
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<Long> cq = cb.createQuery(Long.class);
Root<T> root = cq.from(cls);
cq.select(cb.count(root));
return em.createQuery(cq).getSingleResult();
}
cq.select(root).where(p).orderBy(cb.asc(root.get(JpaObject.id_FIELDNAME)));
return em.createQuery(cq).setMaxResults(size).getResultList();
}
/**
* 根据设置的模式不同输出需要dump的entity className
*
* @return
* @throws Exception
*/
@SuppressWarnings("unchecked")
private List<String> entities() throws Exception {
List<String> list = new ArrayList<>();
if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) {
list.addAll((List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES));
return list;
private Integer count() {
return catalog.values().stream().mapToInt(Integer::intValue).sum();
}
for (String str : (List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES)) {
Class<?> cls = Class.forName(str);
private <T> List<T> list(EntityManager em, Class<T> cls, String id, Integer size) {
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<T> cq = cb.createQuery(cls);
Root<T> root = cq.from(cls);
Predicate p = cb.conjunction();
if (StringUtils.isNotEmpty(id)) {
p = cb.greaterThan(root.get(JpaObject.id_FIELDNAME), id);
}
cq.select(root).where(p).orderBy(cb.asc(root.get(JpaObject.id_FIELDNAME)));
return em.createQuery(cq).setMaxResults(size).getResultList();
}
private <T> void dump(Class<T> cls, EntityManager em, StorageMappings storageMappings, long total)
throws Exception {
// 创建最终存储文件的目录
Path directory = dir.resolve(cls.getName());
Files.createDirectories(directory);
PathUtils.cleanDirectory(directory);
ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
if (Objects.equals(containerEntity.reference(), Reference.strong)) {
list.add(str);
String id = "";
List<T> list = null;
int count = 0;
int loop = 1;
int btach = (int) Math.ceil((total + 0.0) / containerEntity.dumpSize());
do {
list = list(em, cls, id, containerEntity.dumpSize());
if (ListTools.isNotEmpty(list)) {
count = count + list.size();
id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME);
if (StorageObject.class.isAssignableFrom(cls)) {
Path sub = directory.resolve(count + "");
Files.createDirectories(sub);
binary(list, sub, storageMappings);
}
Files.write(directory.resolve(count + ".json"),
pureGsonDateFormated.toJson(list).getBytes(StandardCharsets.UTF_8));
logger.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName());
}
em.clear();
} while (ListTools.isNotEmpty(list));
catalog.put(cls.getName(), count);
}
private <T> void binary(List<T> list, Path sub, StorageMappings storageMappings) throws Exception {
for (T t : list) {
StorageObject s = (StorageObject) t;
String name = s.getStorage();
StorageMapping mapping = storageMappings.get(s.getClass(), name);
if (null == mapping && Config.dumpRestoreStorage().getExceptionInvalidStorage()) {
throw new ExceptionInvalidStorage(s);
}
if (null != mapping) {
Path p = sub.resolve(FilenameUtils.getName(s.path()));
if (s.existContent(mapping)) {
try (OutputStream out = Files.newOutputStream(p)) {
out.write(s.readContent(mapping));
}
}
}
}
}
return list;
}
}
\ No newline at end of file
o2server/x_console/src/main/java/com/x/server/console/action/DumpStorage.java 已删除 100644 → 0
浏览文件 @ dcdbec91
package com.x.server.console.action;
import java.io.File;
import java.io.FileOutputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Objects;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.openjpa.persistence.OpenJPAPersistence;
import com.google.gson.Gson;
import com.x.base.core.container.factory.PersistenceXmlHelper;
import com.x.base.core.entity.JpaObject;
import com.x.base.core.entity.StorageObject;
import com.x.base.core.entity.annotation.ContainerEntity;
import com.x.base.core.entity.annotation.ContainerEntity.Reference;
import com.x.base.core.project.config.Config;
import com.x.base.core.project.config.DumpRestoreData;
import com.x.base.core.project.config.StorageMapping;
import com.x.base.core.project.config.StorageMappings;
import com.x.base.core.project.gson.XGsonBuilder;
import com.x.base.core.project.logger.Logger;
import com.x.base.core.project.logger.LoggerFactory;
import com.x.base.core.project.tools.DateTools;
import com.x.base.core.project.tools.DefaultCharset;
import com.x.base.core.project.tools.ListTools;
public class DumpStorage {
private static Logger logger = LoggerFactory.getLogger(DumpStorage.class);
private Date start = new Date();
private File dir;
private DumpRestoreStorageCatalog catalog;
private Gson pureGsonDateFormated = XGsonBuilder.instance();
public boolean execute(String path) throws Exception {
if (StringUtils.isEmpty(path)) {
this.dir = new File(Config.base(), "local/dump/dumpStorage_" + DateTools.compact(this.start));
} else {
this.dir = new File(path);
if (dir.getAbsolutePath().startsWith(Config.base())) {
logger.print("path can not in base directory.");
return false;
}
}
FileUtils.forceMkdir(this.dir);
this.catalog = new DumpRestoreStorageCatalog();
List<String> storageContainerEntityNames = this.entities();
List<String> classNames = ListTools.includesExcludesWildcard(storageContainerEntityNames,
Config.dumpRestoreStorage().getIncludes(), Config.dumpRestoreStorage().getExcludes());
logger.print("dump storage find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
StorageMappings storageMappings = Config.storageMappings();
File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml");
PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames);
for (int i = 0; i < classNames.size(); i++) {
Class<StorageObject> cls = (Class<StorageObject>) Class.forName(classNames.get(i));
EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
EntityManager em = emf.createEntityManager();
try {
logger.print("dump storage({}/{}): {}, estimate count: {}, estimate size: {}M.", (i + 1),
classNames.size(), cls.getName(), this.estimateCount(em, cls),
(this.estimateSize(em, cls) / 1024 / 1024));
this.dump(cls, em, storageMappings);
} finally {
em.close();
emf.close();
}
}
FileUtils.write(new File(dir, "catalog.json"), XGsonBuilder.instance().toJson(this.catalog),
DefaultCharset.charset);
logger.print(
"dump storage completed, directory: {}, count: {}, normal: {}, empty: {}, invalidStorage: {}, size: {}M, elapsed: {} minutes.",
dir.getAbsolutePath(), this.count(), this.normal(), this.empty(), this.invalidStorage(),
(this.size() / 1024 / 1024), (System.currentTimeMillis() - start.getTime()) / 1000 / 60);
return true;
}
private Integer count() {
return this.catalog.values().stream().mapToInt(DumpRestoreStorageCatalogItem::getCount).sum();
}
private Long size() {
return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getSize).sum();
}
private Long normal() {
return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getNormal).sum();
}
private Long empty() {
return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getEmpty).sum();
}
private Long invalidStorage() {
return this.catalog.values().stream().mapToLong(DumpRestoreStorageCatalogItem::getInvalidStorage).sum();
}
private <T> long estimateCount(EntityManager em, Class<T> cls) {
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<Long> cq = cb.createQuery(Long.class);
Root<T> root = cq.from(cls);
cq.select(cb.count(root));
return em.createQuery(cq).getSingleResult();
}
private <T> long estimateSize(EntityManager em, Class<T> cls) {
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<Long> cq = cb.createQuery(Long.class);
Root<T> root = cq.from(cls);
cq.select(root.get("length"));
List<Long> list = em.createQuery(cq).getResultList();
if (!list.isEmpty()) {
/** 上面的语句有可能返回的是null值,所以要先过滤 */
return list.stream().filter(o -> null != o).mapToLong(Long::longValue).sum();
} else {
return 0L;
}
}
private <T extends StorageObject> void dump(Class<T> cls, EntityManager em, StorageMappings storageMappings)
throws Exception {
/** 创建最终存储文件的目录 */
File classDirectory = new File(dir, cls.getName());
FileUtils.forceMkdir(classDirectory);
FileUtils.cleanDirectory(classDirectory);
int count = 0;
long size = 0L;
int normal = 0;
int invalidStorage = 0;
int empty = 0;
String id = "";
String name = "";
List<T> list = null;
File directory = null;
StorageMapping mapping = null;
List<T> normalList = null;
List<T> emptyList = null;
List<T> invalidStorageList = null;
ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
do {
list = this.list(em, cls, id, containerEntity.dumpSize());
if (ListTools.isNotEmpty(list)) {
count += list.size();
directory = new File(classDirectory, Integer.toString(count));
FileUtils.forceMkdir(directory);
FileUtils.cleanDirectory(directory);
normalList = new ArrayList<T>();
emptyList = new ArrayList<T>();
invalidStorageList = new ArrayList<T>();
for (T t : list) {
name = t.getStorage();
mapping = storageMappings.get(cls, name);
if (StringUtils.isNotEmpty(name)) {
if (null == mapping && Config.dumpRestoreStorage().getExceptionInvalidStorage()) {
throw new Exception("can not find storageMapping class: " + cls.getName() + ", storage: "
+ name + ", id: " + t.getId() + ", name: " + t.getName()
+ ", set exceptionInvalidStorage to false will ignore item.");
}
}
if (null != mapping) {
File file = new File(directory, FilenameUtils.getName(t.path()));
if (t.existContent(mapping)) {
try (FileOutputStream fos = new FileOutputStream(file)) {
size += t.readContent(mapping, fos);
normalList.add(t);
normal++;
}
} else {
emptyList.add(t);
empty++;
}
} else {
invalidStorageList.add(t);
invalidStorage++;
}
}
id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME);
File file = new File(classDirectory, count + ".json");
this.dumpWrite(file, normalList, emptyList, invalidStorageList);
}
em.clear();
} while (ListTools.isNotEmpty(list));
DumpRestoreStorageCatalogItem item = new DumpRestoreStorageCatalogItem();
item.setCount(count);
item.setNormal(normal);
item.setEmpty(empty);
item.setSize(size);
item.setInvalidStorage(invalidStorage);
this.catalog.put(cls.getName(), item);
logger.print(
"dumped storage: " + cls.getName() + ", count: " + count + ", normal: " + normal + ", invalidStorage: "
+ invalidStorage + ", empty: " + empty + ", size: " + (size / 1024 / 1024) + "M.");
}
private <T extends StorageObject> void dumpWrite(File file, List<T> normalList, List<T> emptyList,
List<T> invalidStorageList) throws Exception {
LinkedHashMap<String, List<T>> o = new LinkedHashMap<>();
o.put("normals", normalList);
o.put("emptys", emptyList);
o.put("invalidStorages", invalidStorageList);
FileUtils.write(file, pureGsonDateFormated.toJson(o), DefaultCharset.charset);
}
private <T extends StorageObject> List<T> list(EntityManager em, Class<T> cls, String id, Integer size) {
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<T> cq = cb.createQuery(cls);
Root<T> root = cq.from(cls);
Predicate p = cb.conjunction();
if (StringUtils.isNotEmpty(id)) {
p = cb.greaterThan(root.get("id"), id);
}
cq.select(root).where(p).orderBy(cb.asc(root.get("id")));
return em.createQuery(cq).setMaxResults(size).getResultList();
}
/**
* 根据设置的模式不同输出需要dump的entity className
*
* @return
* @throws Exception
*/
@SuppressWarnings("unchecked")
private List<String> entities() throws Exception {
List<String> list = new ArrayList<>();
if (StringUtils.equals(Config.dumpRestoreData().getMode(), DumpRestoreData.TYPE_FULL)) {
list.addAll((List<String>) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES));
return list;
}
for (String str : (List<String>) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES)) {
Class<?> cls = Class.forName(str);
ContainerEntity containerEntity = cls.getAnnotation(ContainerEntity.class);
if (Objects.equals(containerEntity.reference(), Reference.strong)) {
list.add(str);
}
}
return list;
}
public static class Item {
}
}
\ No newline at end of file
o2server/x_console/src/main/java/com/x/server/console/action/ExceptionDirectoryNotExist.java 0 → 100644
浏览文件 @ e753e163
package com.x.server.console.action;
import java.nio.file.Path;
import com.x.base.core.project.exception.PromptException;
class ExceptionDirectoryNotExist extends PromptException {
private static final long serialVersionUID = -5515077418025884395L;
ExceptionDirectoryNotExist(Path path) {
super("directory not exist: {}.", path.toString());
}
}
o2server/x_console/src/main/java/com/x/server/console/action/ExceptionFileNotExist.java 0 → 100644
浏览文件 @ e753e163
package com.x.server.console.action;
import java.nio.file.Path;
import com.x.base.core.project.exception.PromptException;
class ExceptionFileNotExist extends PromptException {
private static final long serialVersionUID = -5515077418025884395L;
ExceptionFileNotExist(Path path) {
super("file not exist: {}.", path.toString());
}
}
o2server/x_console/src/main/java/com/x/server/console/action/ExceptionInvalidStorage.java 0 → 100644
浏览文件 @ e753e163
package com.x.server.console.action;
import com.x.base.core.entity.StorageObject;
import com.x.base.core.project.exception.PromptException;
class ExceptionInvalidStorage extends PromptException {
private static final long serialVersionUID = -5515077418025884395L;
ExceptionInvalidStorage(StorageObject storageObject) {
super("can not find storageMapping class: " + storageObject.getClass().getName() + ", storage: "
+ storageObject.getStorage() + ", id: " + storageObject.getId() + ", name: " + storageObject.getName()
+ ", set exceptionInvalidStorage to false will ignore item.");
}
}
o2server/x_console/src/main/java/com/x/server/console/action/ExceptionMappingNotExist.java 0 → 100644
浏览文件 @ e753e163
package com.x.server.console.action;
import com.x.base.core.project.exception.PromptException;
class ExceptionMappingNotExist extends PromptException {
private static final long serialVersionUID = -5515077418025884395L;
ExceptionMappingNotExist() {
super("can not find storageMapping");
}
}
o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java
浏览文件 @ e753e163
package com.x.server.console.action;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.FlushModeType;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.openjpa.persistence.OpenJPAPersistence;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.x.base.core.container.factory.PersistenceXmlHelper;
import com.x.base.core.entity.JpaObject;
import com.x.base.core.entity.StorageObject;
import com.x.base.core.entity.annotation.ContainerEntity;
import com.x.base.core.project.config.Config;
import com.x.base.core.project.config.StorageMapping;
import com.x.base.core.project.config.StorageMappings;
import com.x.base.core.project.gson.XGsonBuilder;
import com.x.base.core.project.logger.Logger;
import com.x.base.core.project.logger.LoggerFactory;
import com.x.base.core.project.tools.BaseTools;
import com.x.base.core.project.tools.DateTools;
import com.x.base.core.project.tools.DefaultCharset;
import com.x.base.core.project.tools.ListTools;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.openjpa.persistence.OpenJPAPersistence;
/**
* @author zhourui
*/
public class RestoreData {
private static Logger logger = LoggerFactory.getLogger(RestoreData.class);
private Date start = new Date();
private File dir;
private DumpRestoreDataCatalog catalog;
private Gson pureGsonDateFormated = XGsonBuilder.instance();
public boolean execute(String path) throws Exception {
Date start = new Date();
Path dir;
if (StringUtils.isEmpty(path)) {
logger.print("path is empty.");
logger.warn("path is empty.");
}
if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) {
this.dir = new File(Config.base(), "local/dump/dumpData_" + path);
this.catalog = BaseTools.readConfigObject("local/dump/dumpData_" + path + "/catalog.json",
DumpRestoreDataCatalog.class);
dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + path);
} else {
this.dir = new File(path);
if (!(this.dir.exists() && this.dir.isDirectory())) {
logger.print("dir not exist: {}.", path);
dir = Paths.get(path);
if ((!Files.exists(dir)) || (!Files.isDirectory(dir))) {
logger.warn("directory not exist: {}.", path);
return false;
} else if (StringUtils.startsWith(dir.getAbsolutePath(), Config.base())) {
logger.print("path can not in base directory.");
} else if (dir.startsWith(Paths.get(Config.base()))) {
logger.warn("path can not in base directory.");
return false;
}
}
this.catalog = XGsonBuilder.instance().fromJson(
FileUtils.readFileToString(new File(dir, "catalog.json"), DefaultCharset.charset_utf_8),
DumpRestoreDataCatalog.class);
return this.execute();
Thread thread = new Thread(new RunnableImpl(dir, start));
thread.start();
return true;
}
@SuppressWarnings("unchecked")
public boolean execute() throws Exception {
List<String> containerEntityNames = new ArrayList<>();
containerEntityNames.addAll((List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES));
List<String> classNames = new ArrayList<>();
classNames.addAll(this.catalog.keySet());
classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(),
Config.dumpRestoreData().getExcludes());
classNames = ListTools.includesExcludesWildcard(containerEntityNames, classNames, null);
logger.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.getAbsolutePath());
File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml");
PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames);
long count = 0;
for (int i = 0; i < classNames.size(); i++) {
Class<JpaObject> cls = (Class<JpaObject>) Class.forName(classNames.get(i));
public class RunnableImpl implements Runnable {
private Path dir;
private Date start;
private DumpRestoreDataCatalog catalog;
private Gson gson;
public RunnableImpl(Path dir, Date start) throws IOException {
this.dir = dir;
this.start = start;
this.catalog = new DumpRestoreDataCatalog();
this.gson = XGsonBuilder.instance();
Path path = dir.resolve("catalog.json");
this.catalog = XGsonBuilder.instance().fromJson(
new String(Files.readAllBytes(path), StandardCharsets.UTF_8), DumpRestoreDataCatalog.class);
}
@Override
public void run() {
try {
List<String> classNames = this.entities();
logger.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.toString());
Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(),
DateTools.compact(start) + "_restore.xml");
PersistenceXmlHelper.write(xml.toString(), classNames);
Stream<String> stream = BooleanUtils.isTrue(Config.dumpRestoreData().getParallel())
? classNames.parallelStream()
: classNames.stream();
AtomicInteger idx = new AtomicInteger(1);
AtomicLong total = new AtomicLong(0);
stream.forEach(className -> {
String nameOfThread = Thread.currentThread().getName();
try {
Thread.currentThread().setName(RestoreData.class.getName() + ":" + className);
@SuppressWarnings("unchecked")
Class<JpaObject> cls = (Class<JpaObject>) Class.forName(className);
logger.print("restore data({}/{}): {}.", idx.getAndAdd(1), classNames.size(), cls.getName());
long size = restore(cls, xml);
total.getAndAdd(size);
} catch (Exception e) {
logger.error(new Exception(String.format("restore:%s error.", className), e));
} finally {
Thread.currentThread().setName(nameOfThread);
}
});
logger.print("restore data completed, directory: {}, count: {}, total: {}, elapsed: {} minutes.",
dir.toString(), idx.get(), total.longValue(),
(System.currentTimeMillis() - start.getTime()) / 1000 / 60);
} catch (Exception e) {
logger.error(e);
}
}
@SuppressWarnings("unchecked")
private List<String> entities() throws Exception {
List<String> containerEntityNames = new ArrayList<>();
containerEntityNames.addAll((List<String>) Config.resource(Config.RESOURCE_CONTAINERENTITYNAMES));
List<String> classNames = new ArrayList<>();
classNames.addAll(this.catalog.keySet());
classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(),
Config.dumpRestoreData().getExcludes());
return ListTools.includesExcludesWildcard(containerEntityNames, classNames, null);
}
private long restore(Class<?> cls, Path xml) throws Exception {
EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
if (emf != null) {
EntityManager em = emf.createEntityManager();
em.setFlushMode(FlushModeType.COMMIT);
try {
logger.print("restore data({}/{}): {}, count: {}.", (i + 1), classNames.size(), cls.getName(),
catalog.get(cls.getName()));
count = count + this.store(cls, em);
} finally {
em.close();
emf.close();
xml.getFileName().toString(),
PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
AtomicLong count = new AtomicLong(0);
AtomicInteger batch = new AtomicInteger(1);
try {
Path directory = dir.resolve(cls.getName());
if ((!Files.exists(directory)) || (!Files.isDirectory(directory))) {
throw new ExceptionDirectoryNotExist(directory);
}
} else {
logger.warn("can not create 'EntityManagerFactory' for Entity:[" + cls.getName() + "]");
StorageMappings storageMappings = Config.storageMappings();
this.clean(cls, emf, storageMappings, cls.getAnnotation(ContainerEntity.class));
List<Path> paths = this.list(directory);
paths.stream().forEachOrdered(o -> {
logger.print("restore {}/{} part of data:{}.", batch.getAndAdd(1), paths.size(), cls.getName());
EntityManager em = null;
try {
em = emf.createEntityManager();
em.getTransaction().begin();
JsonArray raws = this.convert(o);
for (JsonElement json : raws) {
Object t = gson.fromJson(json, cls);
if (StorageObject.class.isAssignableFrom(cls)) {
Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString()));
this.binary(t, cls, sub, storageMappings);
}
em.persist(t);
count.getAndAdd(1);
}
em.getTransaction().commit();
em.clear();
} catch (Exception e) {
logger.error(new Exception(String.format("restore error with file:%s.", o.toString()), e));
} finally {
em.close();
}
});
logger.print("restore data: {} completed, count: {}.", cls.getName(), count.intValue());
} catch (Exception e) {
logger.error(e);
} finally {
emf.close();
}
return count.longValue();
}
logger.print("restore data completed, total count: {}, elapsed: {} minutes.", count,
(System.currentTimeMillis() - start.getTime()) / 1000 / 60);
return true;
}
private <T> long store(Class<T> cls, EntityManager em) throws Exception {
File directory = new File(this.dir, cls.getName());
if ((!directory.exists()) || (!directory.isDirectory())) {
throw new Exception("can not find directory: " + directory.getAbsolutePath() + ".");
private List<Path> list(Path directory) throws IOException {
List<Path> list = new ArrayList<>();
try (Stream<Path> stream = Files.list(directory)) {
stream.filter(p -> StringUtils.endsWithIgnoreCase(p.getFileName().toString(), ".json"))
.sorted((Path p1, Path p2) -> {
Integer i1 = Integer.parseInt(FilenameUtils.getBaseName(p1.getFileName().toString()));
Integer i2 = Integer.parseInt(FilenameUtils.getBaseName(p2.getFileName().toString()));
return i1.compareTo(i2);
}).forEach(list::add);
}
return list;
}
long count = 0;
List<File> files = new ArrayList<>(FileUtils.listFiles(directory, new String[] { "json" }, false));
/** 对文件进行排序,和dump的时候的顺序保持一直 */
Collections.sort(files, new Comparator<File>() {
public int compare(File o1, File o2) {
String n1 = FilenameUtils.getBaseName(o1.getName());
String n2 = FilenameUtils.getBaseName(o2.getName());
Integer i1 = Integer.parseInt(n1);
Integer i2 = Integer.parseInt(n2);
return i1.compareTo(i2);
@SuppressWarnings("unchecked")
private void binary(Object o, Class<?> cls, Path sub, StorageMappings storageMappings) throws Exception {
StorageObject so = (StorageObject) o;
StorageMapping mapping = null;
if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistribute())) {
mapping = storageMappings.random((Class<StorageObject>) cls);
} else {
mapping = storageMappings.get((Class<StorageObject>) cls, so.getStorage());
}
if (null == mapping) {
throw new ExceptionMappingNotExist();
}
Path path = sub.resolve(Paths.get(so.path()).getFileName());
if (!Files.exists(path)) {
throw new ExceptionFileNotExist(path);
}
});
/** 尽量在最后进行清空操作 */
this.clean(cls, em);
File file = null;
for (int i = 0; i < files.size(); i++) {
file = files.get(i);
logger.print("restoring {}/{} part of data:{}.", (i + 1), files.size(), cls.getName());
JsonArray raws = this.convert(file);
em.getTransaction().begin();
for (JsonElement o : raws) {
T t = pureGsonDateFormated.fromJson(o, cls);
em.persist(t);
count++;
try (InputStream input = Files.newInputStream(path)) {
so.saveContent(mapping, input, so.getName());
}
em.getTransaction().commit();
em.clear();
}
System.out.println("restore data: " + cls.getName() + " completed, count: " + count + ".");
return count;
}
private JsonArray convert(File file) throws Exception {
/* 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double */
String json = FileUtils.readFileToString(file, DefaultCharset.charset);
JsonElement jsonElement = pureGsonDateFormated.fromJson(json, JsonElement.class);
return jsonElement.getAsJsonArray();
}
private JsonArray convert(Path path) throws IOException {
// 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double
String json = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
JsonElement jsonElement = gson.fromJson(json, JsonElement.class);
return jsonElement.getAsJsonArray();
}
private <T> void clean(Class<T> cls, EntityManager em) throws Exception {
List<T> list = null;
do {
if (ListTools.isNotEmpty(list)) {
em.getTransaction().begin();
for (T t : list) {
em.remove(t);
private <T> void clean(Class<T> cls, EntityManagerFactory emf, StorageMappings storageMappings,
ContainerEntity containerEntity) throws Exception {
EntityManager em = emf.createEntityManager();
List<T> list = null;
do {
if (ListTools.isNotEmpty(list)) {
em.getTransaction().begin();
for (T t : list) {
em.remove(t);
if (StorageObject.class.isAssignableFrom(cls)) {
StorageObject so = (StorageObject) t;
@SuppressWarnings("unchecked")
StorageMapping mapping = storageMappings.get((Class<StorageObject>) cls, so.getStorage());
if (null != mapping) {
so.deleteContent(mapping);
}
}
}
em.getTransaction().commit();
}
em.getTransaction().commit();
}
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<T> cq = cb.createQuery(cls);
Root<T> root = cq.from(cls);
cq.select(root);
list = em.createQuery(cq).setMaxResults(Config.dumpRestoreData().getBatchSize()).getResultList();
} while (ListTools.isNotEmpty(list));
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<T> cq = cb.createQuery(cls);
Root<T> root = cq.from(cls);
list = em.createQuery(cq.select(root)).setMaxResults(containerEntity.dumpSize()).getResultList();
} while (ListTools.isNotEmpty(list));
}
}
}
\ No newline at end of file
o2server/x_console/src/main/java/com/x/server/console/action/RestoreStorage.java 已删除 100644 → 0
浏览文件 @ dcdbec91
package com.x.server.console.action;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import javax.persistence.EntityManager;
import javax.persistence.EntityManagerFactory;
import javax.persistence.FlushModeType;
import javax.persistence.criteria.CriteriaBuilder;
import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Root;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.openjpa.persistence.OpenJPAPersistence;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.x.base.core.container.factory.PersistenceXmlHelper;
import com.x.base.core.entity.StorageObject;
import com.x.base.core.project.config.Config;
import com.x.base.core.project.config.StorageMapping;
import com.x.base.core.project.config.StorageMappings;
import com.x.base.core.project.gson.XGsonBuilder;
import com.x.base.core.project.logger.Logger;
import com.x.base.core.project.logger.LoggerFactory;
import com.x.base.core.project.tools.BaseTools;
import com.x.base.core.project.tools.DateTools;
import com.x.base.core.project.tools.DefaultCharset;
import com.x.base.core.project.tools.ListTools;
/**
* @author zhourui
*/
public class RestoreStorage {
private static Logger logger = LoggerFactory.getLogger(RestoreStorage.class);
private Date start = new Date();
private File dir;
private DumpRestoreStorageCatalog catalog;
private Gson pureGsonDateFormated = XGsonBuilder.instance();
public boolean execute(final String path) throws Exception {
if (StringUtils.isEmpty(path)) {
logger.print("path is empty.");
}
if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) {
this.dir = new File(Config.base(), "local/dump/dumpStorage_" + path);
this.catalog = BaseTools.readConfigObject("local/dump/dumpStorage_" + path + "/catalog.json",
DumpRestoreStorageCatalog.class);
} else {
this.dir = new File(path);
if (!(this.dir.exists() && this.dir.isDirectory())) {
logger.print("dir not exist: {}.", path);
return false;
} else if (StringUtils.startsWith(dir.getAbsolutePath(), Config.base())) {
logger.print("path can not in base directory.");
return false;
}
this.catalog = XGsonBuilder.instance().fromJson(FileUtils
.readFileToString(new File(dir.getAbsolutePath(), "catalog.json"), DefaultCharset.charset_utf_8),
DumpRestoreStorageCatalog.class);
}
return this.execute();
}
private boolean execute() throws Exception {
final List<String> storageContainerEntityNames = new ArrayList<>();
storageContainerEntityNames.addAll((List<String>) Config.resource(Config.RESOURCE_STORAGECONTAINERENTITYNAMES));
List<String> classNames = new ArrayList<>();
classNames.addAll(this.catalog.keySet());
classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreStorage().getIncludes(),
Config.dumpRestoreStorage().getExcludes());
logger.print("restore storage find {} to restore.", classNames.size());
final File persistence = new File(Config.dir_local_temp_classes(), DateTools.compact(this.start) + "_dump.xml");
PersistenceXmlHelper.write(persistence.getAbsolutePath(), classNames);
final StorageMappings storageMappings = Config.storageMappings();
int count = 0;
for (int i = 0; i < classNames.size(); i++) {
final Class<StorageObject> cls = (Class<StorageObject>) Class.forName(classNames.get(i));
final EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
persistence.getName(), PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
final EntityManager em = emf.createEntityManager();
em.setFlushMode(FlushModeType.COMMIT);
try {
final DumpRestoreStorageCatalogItem item = this.catalog.get(cls.getName());
logger.print(
"restore storage({}/{}): {}, count: {}, normal: {} will be restore, invalidStorage: {} and empty: {} will be ignore, size: {}M.",
(i + 1), classNames.size(), cls.getName(), item.getCount(), item.getNormal(),
item.getInvalidStorage(), item.getEmpty(), (item.getSize() / 1024 / 1024));
count += this.store(cls, em, storageMappings);
} finally {
em.close();
emf.close();
}
logger.print("restore storage completed, total count: {}, elapsed: {} minutes.", count,
(System.currentTimeMillis() - start.getTime()) / 1000 / 60);
}
return false;
}
private <T extends StorageObject> long store(final Class<T> cls, final EntityManager em,
final StorageMappings storageMappings) throws Exception {
final File classDirectory = new File(this.dir, cls.getName());
if ((!classDirectory.exists()) || (!classDirectory.isDirectory())) {
throw new Exception("can not find directory: " + classDirectory.getAbsolutePath() + ".");
}
long count = 0;
final List<File> files = new ArrayList<File>(
FileUtils.listFiles(classDirectory, new String[] { "json" }, false));
/** 对文件进行排序,和dump的时候的顺序保持一直 */
Collections.sort(files, new Comparator<File>() {
public int compare(final File o1, final File o2) {
final String n1 = FilenameUtils.getBaseName(o1.getName());
final String n2 = FilenameUtils.getBaseName(o2.getName());
final Integer i1 = Integer.parseInt(n1);
final Integer i2 = Integer.parseInt(n2);
return i1.compareTo(i2);
}
});
/** 尽量将删除的工作放在后面 */
this.clean(cls, em, storageMappings);
StorageMapping mapping = null;
File file = null;
for (int i = 0; i < files.size(); i++) {
file = files.get(i);
/** 必须先转换成 jsonElement 不能直接转成泛型T,如果直接转会有类型不匹配比如Integer变成了Double */
logger.print("restoring " + (i + 1) + "/" + files.size() + " part of storage: " + cls.getName() + ".");
final JsonArray raws = this.convert(file);
if (null != raws) {
em.getTransaction().begin();
for (final JsonElement o : raws) {
final T t = pureGsonDateFormated.fromJson(o, cls);
if (Config.dumpRestoreStorage().getRedistribute()) {
mapping = storageMappings.random(cls);
} else {
mapping = storageMappings.get(cls, t.getStorage());
}
if (null == mapping) {
throw new Exception(
"can not find storageMapping class: " + cls.getName() + ", name:" + t.getName());
}
final File source = new File(classDirectory, FilenameUtils.getBaseName(file.getName())
+ StorageObject.PATHSEPARATOR + FilenameUtils.getName(t.path()));
try (FileInputStream input = new FileInputStream(source)) {
t.saveContent(mapping, input, t.getName());
}
em.persist(t);
count++;
}
em.getTransaction().commit();
em.clear();
Runtime.getRuntime().gc();
}
}
return count;
}
private JsonArray convert(final File file) throws IOException {
/** 这里不进行判断,因为格式是严格约定的,出现意外应该先报错停止 */
final String json = FileUtils.readFileToString(file, DefaultCharset.charset);
final JsonElement jsonElement = pureGsonDateFormated.fromJson(json, JsonElement.class);
final JsonObject jsonObject = jsonElement.getAsJsonObject();
return jsonObject.get("normals").getAsJsonArray();
}
private <T extends StorageObject> void clean(final Class<T> cls, final EntityManager em,
final StorageMappings storageMappings) throws Exception {
List<T> list = null;
StorageMapping mapping = null;
do {
if (ListTools.isNotEmpty(list)) {
em.getTransaction().begin();
for (final T t : list) {
mapping = storageMappings.get(cls, t.getStorage());
if (null != mapping) {
t.deleteContent(mapping);
}
em.remove(t);
}
em.getTransaction().commit();
}
final CriteriaBuilder cb = em.getCriteriaBuilder();
final CriteriaQuery<T> cq = cb.createQuery(cls);
final Root<T> root = cq.from(cls);
cq.select(root);
list = em.createQuery(cq).setMaxResults(Config.dumpRestoreData().getBatchSize()).getResultList();
} while (ListTools.isNotEmpty(list));
}
}
\ No newline at end of file
o2server/x_file_core_entity/src/main/java/com/x/file/core/entity/personal/Attachment.java
浏览文件 @ e753e163
......@@ -36,7 +36,7 @@ import com.x.base.core.project.annotation.FieldDescribe;
import com.x.base.core.project.tools.DateTools;
import com.x.file.core.entity.PersistenceProperties;
@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@Entity
@Table(name = PersistenceProperties.Personal.Attachment.table, uniqueConstraints = {
@UniqueConstraint(name = PersistenceProperties.Personal.Attachment.table + JpaObject.IndexNameMiddle
......
o2server/x_meeting_core_entity/src/main/java/com/x/meeting/core/entity/Attachment.java
浏览文件 @ e753e163
......@@ -23,7 +23,7 @@ import com.x.base.core.entity.annotation.CheckPersist;
import com.x.base.core.entity.annotation.ContainerEntity;
import com.x.base.core.project.annotation.FieldDescribe;
@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@Entity
@Table(name = PersistenceProperties.Attachment.table, uniqueConstraints = {
@UniqueConstraint(name = PersistenceProperties.Attachment.table + JpaObject.IndexNameMiddle
......
o2server/x_message_core_entity/src/main/java/com/x/message/core/entity/IMMsgFile.java
浏览文件 @ e753e163
......@@ -20,7 +20,7 @@ import java.util.Date;
*/
@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@Entity
@Table(name = PersistenceProperties.IMMsgFile.table, uniqueConstraints = {
@UniqueConstraint(name = PersistenceProperties.IMMsgFile.table + JpaObject.IndexNameMiddle
......
o2server/x_okr_core_entity/src/main/java/com/x/okr/entity/OkrAttachmentFileInfo.java
浏览文件 @ e753e163
......@@ -30,7 +30,7 @@ import com.x.base.core.project.tools.DateTools;
* 附件文件信息管理表
*
*/
@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@Entity
@Table(name = PersistenceProperties.OkrAttachmentFileInfo.table, uniqueConstraints = {
@UniqueConstraint(name = PersistenceProperties.OkrAttachmentFileInfo.table + JpaObject.IndexNameMiddle
......
o2server/x_processplatform_core_entity/src/main/java/com/x/processplatform/core/entity/content/Attachment.java
浏览文件 @ e753e163
......@@ -38,7 +38,7 @@ import com.x.base.core.project.tools.DateTools;
import com.x.processplatform.core.entity.PersistenceProperties;
import com.x.processplatform.core.entity.element.ActivityType;
@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@ContainerEntity(dumpSize = 5, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@Entity
@Table(name = PersistenceProperties.Content.Attachment.table, uniqueConstraints = {
@UniqueConstraint(name = PersistenceProperties.Content.Attachment.table + JpaObject.IndexNameMiddle
......
o2server/x_program_center_core_entity/src/main/java/com/x/program/center/core/entity/Structure.java
浏览文件 @ e753e163
......@@ -25,7 +25,7 @@ import com.x.base.core.project.annotation.FieldDescribe;
import com.x.base.core.project.tools.StringTools;
@Entity
@ContainerEntity(dumpSize = 1000, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@ContainerEntity(dumpSize = 10, type = ContainerEntity.Type.content, reference = ContainerEntity.Reference.strong)
@Table(name = PersistenceProperties.Structure.table, uniqueConstraints = {
@UniqueConstraint(name = PersistenceProperties.Structure.table + JpaObject.IndexNameMiddle
+ JpaObject.DefaultUniqueConstraintSuffix, columnNames = { JpaObject.IDCOLUMN,
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或