Skip to content

o2oa
o2oa

ghsby / o2oa
落后 Fork 源项目 2880 个版本

Fork自 浙江兰德纵横网络技术股份有限公司 / o2oa

通知 1
Star 1
Fork 0
o2oa
o2oa

体验新版 GitCode，发现更多精彩内容 >>

提交 31897403 编写于 作者: Z zhourui
浏览文件
操作

增加dumpData,restoreData 中attachStorage配置

上级 795308de
隐藏空白更改
内联 并排
Showing with 118 addition and 89 deletion
o2server/x_base_core_project/src/main/java/com/x/base/core/project/config/DumpRestoreData.java
浏览文件 @ 31897403
......@@ -17,35 +17,31 @@ public class DumpRestoreData extends ConfigObject {
return new DumpRestoreData();
}
public static final String TYPE_FULL = "full";
public static final String TYPE_LITE = "lite";
public static final String MODE_FULL = "full";
public static final String MODE_LITE = "lite";
public static final String RESTOREOVERRIDE_CLEAN = "clean";
public static final String RESTOREOVERRIDE_SKIPEXISTED = "skipExisted";
public static final String DEFAULT_TYPE = TYPE_LITE;
public static final Boolean DEFAULT_PARALLEL = true;
public static final Boolean DEFAULT_REDISTRIBUTE = true;
public static final Boolean DEFAULT_EXCEPTIONINVALIDSTORAGE = true;
public static final Boolean DEFAULT_ATTACHSTORAGE = true;
public static final String DEFAULT_ITEMCATEGORY = "";
public DumpRestoreData() {
this.enable = false;
this.includes = new ArrayList<>();
this.excludes = new ArrayList<>();
this.mode = DEFAULT_TYPE;
this.mode = MODE_LITE;
this.parallel = DEFAULT_PARALLEL;
this.redistribute = DEFAULT_REDISTRIBUTE;
this.attachStorage = DEFAULT_ATTACHSTORAGE;
this.exceptionInvalidStorage = DEFAULT_EXCEPTIONINVALIDSTORAGE;
this.itemCategory = DEFAULT_ITEMCATEGORY;
}
@FieldDescribe("是否启用.")
private Boolean enable;
@FieldDescribe("导出导入包含对象,可以使用通配符*.")
@FieldDescribe("导出导入包含对象,可以使用通配符*,如仅导出待办数据:com.x.processplatform.core.entity.content.Task.")
private List<String> includes;
@FieldDescribe("导出导入排除对象,可以使用通配符*.")
@FieldDescribe("导出导入排除对象,可以使用通配符*,如不导出流程实例数据com.x.processplatform.core.entity.content.*")
private List<String> excludes;
@FieldDescribe("导出数据模式,lite|full,默认使用lite")
......@@ -54,31 +50,41 @@ public class DumpRestoreData extends ConfigObject {
@FieldDescribe("使用并行导出,默认true")
private Boolean parallel;
@FieldDescribe("是否进行重新分布.")
private Boolean redistribute;
@FieldDescribe("无法获取storage是否升起错误.")
private Boolean exceptionInvalidStorage;
@FieldDescribe("无法获取storage是否升起错误.")
private Boolean storageXXXXXXXXXXXXXXXX;
@FieldDescribe("是否同时导入导出storage中存储的文件.")
private Boolean attachStorage;
@FieldDescribe("是否进行重新分布.")
private Boolean redistributeStorage;
public Boolean getRedistributeStorage() {
return redistributeStorage;
}
public void setRedistributeStorage(Boolean redistributeStorage) {
this.redistributeStorage = redistributeStorage;
}
public Boolean getAttachStorage() {
return attachStorage;
}
public void setAttachStorage(Boolean attachStorage) {
this.attachStorage = attachStorage;
}
@FieldDescribe("数据导入方式,clean:清空重新导入,skipExisted:如果有相同id的数据跳过.默认方式为clean.")
private String restoreOverride;
@FieldDescribe("对于com.x.query.core.entity.Item的itemCategory进行单独过滤,可选值pp, cms, bbs, pp_dict.")
private String itemCategory;
public String getItemCategory() {
return this.itemCategory;
}
public Boolean getRedistribute() {
return BooleanUtils.isNotFalse(redistribute);
}
public Boolean getExceptionInvalidStorage() {
return BooleanUtils.isNotFalse(exceptionInvalidStorage);
}
......@@ -88,11 +94,7 @@ public class DumpRestoreData extends ConfigObject {
}
public String getMode() {
return StringUtils.equals(TYPE_FULL, mode) ? TYPE_FULL : TYPE_LITE;
}
public Boolean getEnable() {
return BooleanUtils.isTrue(this.enable);
return StringUtils.equals(MODE_FULL, mode) ? MODE_FULL : MODE_LITE;
}
public List<String> getIncludes() {
......@@ -123,18 +125,10 @@ public class DumpRestoreData extends ConfigObject {
this.parallel = parallel;
}
public void setRedistribute(Boolean redistribute) {
this.redistribute = redistribute;
}
public void setExceptionInvalidStorage(Boolean exceptionInvalidStorage) {
this.exceptionInvalidStorage = exceptionInvalidStorage;
}
public void setEnable(Boolean enable) {
this.enable = enable;
}
public void setMode(String mode) {
this.mode = mode;
}
......
o2server/x_console/src/main/java/com/x/server/console/action/DumpData.java
浏览文件 @ 31897403
......@@ -10,7 +10,9 @@ import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.persistence.EntityManager;
......@@ -36,6 +38,7 @@ import com.x.base.core.entity.dataitem.DataItem;
import com.x.base.core.entity.dataitem.ItemCategory;
import com.x.base.core.entity.tools.JpaObjectTools;
import com.x.base.core.project.config.Config;
import com.x.base.core.project.config.DumpRestoreData;
import com.x.base.core.project.config.StorageMapping;
import com.x.base.core.project.config.StorageMappings;
import com.x.base.core.project.gson.XGsonBuilder;
......@@ -46,7 +49,7 @@ import com.x.base.core.project.tools.ListTools;
public class DumpData {
private static Logger logger = LoggerFactory.getLogger(DumpData.class);
private static final Logger LOGGER = LoggerFactory.getLogger(DumpData.class);
public boolean execute(String path) throws IOException, URISyntaxException {
Path dir = null;
......@@ -56,7 +59,7 @@ public class DumpData {
} else {
dir = Paths.get(path);
if (dir.startsWith(Paths.get(Config.base()))) {
logger.warn("path can not in base directory.");
LOGGER.warn("path can not in base directory.");
return false;
}
}
......@@ -87,8 +90,8 @@ public class DumpData {
public void run() {
try {
List<String> classNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader));
logger.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
List<String> classNames = this.entities(classLoader);
LOGGER.print("find {} data to dump, start at {}.", classNames.size(), DateTools.format(start));
Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(),
DateTools.compact(start) + "_dump.xml");
PersistenceXmlHelper.write(xml.toString(), classNames, true, classLoader);
......@@ -106,18 +109,16 @@ public class DumpData {
try {
@SuppressWarnings("unchecked")
Class<JpaObject> cls = (Class<JpaObject>) classLoader.loadClass(className);
// emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(),
// PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(), xml.getFileName().toString(),
PersistenceXmlHelper.properties(cls.getName(), false));
em = emf.createEntityManager();
long estimateCount = estimateCount(em, cls);
logger.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(),
LOGGER.print("dump data({}/{}): {}, count: {}.", idx.getAndAdd(1), classNames.size(),
cls.getName(), estimateCount);
dump(cls, em, storageMappings, estimateCount);
dump(cls, em, storageMappings, Config.dumpRestoreData().getAttachStorage(), estimateCount);
} catch (Exception e) {
e.printStackTrace();
logger.error(new Exception(String.format("dump:%s error.", className), e));
LOGGER.error(new Exception(String.format("dump:%s error.", className), e));
} finally {
Thread.currentThread().setName(nameOfThread);
if (null != em) {
......@@ -130,7 +131,7 @@ public class DumpData {
});
Files.write(dir.resolve("catalog.json"),
pureGsonDateFormated.toJson(catalog).getBytes(StandardCharsets.UTF_8));
logger.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(),
LOGGER.print("dump data completed, directory: {}, count: {}, elapsed: {} minutes.", dir.toString(),
count(), (System.currentTimeMillis() - start.getTime()) / 1000 / 60);
} catch (Exception e) {
......@@ -138,6 +139,24 @@ public class DumpData {
}
}
private List<String> entities(ClassLoader classLoader) throws Exception {
List<String> classNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader));
classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(),
Config.dumpRestoreData().getExcludes());
if (StringUtils.equalsIgnoreCase(DumpRestoreData.MODE_LITE, Config.dumpRestoreData().getMode())) {
return classNames.stream().filter(o -> {
try {
ContainerEntity containerEntity = classLoader.loadClass(o).getAnnotation(ContainerEntity.class);
return Objects.equals(containerEntity.reference(), ContainerEntity.Reference.strong);
} catch (Exception e) {
LOGGER.error(e);
}
return false;
}).collect(Collectors.toList());
}
return classNames;
}
private <T extends JpaObject> long estimateCount(EntityManager em, Class<T> cls) {
CriteriaBuilder cb = em.getCriteriaBuilder();
CriteriaQuery<Long> cq = cb.createQuery(Long.class);
......@@ -167,8 +186,8 @@ public class DumpData {
return em.createQuery(cq).setMaxResults(size).getResultList();
}
private <T> void dump(Class<T> cls, EntityManager em, StorageMappings storageMappings, long total)
throws Exception {
private <T> void dump(Class<T> cls, EntityManager em, StorageMappings storageMappings, boolean attachStorage,
long total) throws Exception {
// 创建最终存储文件的目录
Path directory = dir.resolve(cls.getName());
Files.createDirectories(directory);
......@@ -184,14 +203,14 @@ public class DumpData {
if (ListTools.isNotEmpty(list)) {
count = count + list.size();
id = BeanUtils.getProperty(list.get(list.size() - 1), JpaObject.id_FIELDNAME);
if (StorageObject.class.isAssignableFrom(cls)) {
if (StorageObject.class.isAssignableFrom(cls) && attachStorage) {
Path sub = directory.resolve(count + "");
Files.createDirectories(sub);
binary(list, sub, storageMappings);
}
Files.write(directory.resolve(count + ".json"),
pureGsonDateFormated.toJson(list).getBytes(StandardCharsets.UTF_8));
logger.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName());
LOGGER.print("dump data {}/{} part of data:{}.", loop++, btach, cls.getName());
}
em.clear();
} while (ListTools.isNotEmpty(list));
......
o2server/x_console/src/main/java/com/x/server/console/action/RestoreData.java
浏览文件 @ 31897403
......@@ -13,6 +13,7 @@ import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.persistence.EntityManager;
......@@ -23,6 +24,7 @@ import javax.persistence.criteria.CriteriaQuery;
import javax.persistence.criteria.Predicate;
import javax.persistence.criteria.Root;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -56,26 +58,29 @@ public class RestoreData {
public boolean execute(String path) throws IOException, URISyntaxException {
Date start = new Date();
Path dir;
if (StringUtils.isEmpty(path)) {
LOGGER.warn("{}.", () -> "path is empty.");
}
ClassLoader classLoader = EntityClassLoaderTools.concreteClassLoader();
Path dir = dir(path);
Thread thread = new Thread(new RunnableImpl(dir, start, classLoader));
thread.start();
return true;
}
private Path dir(String path) throws IOException, URISyntaxException {
Path dir;
if (BooleanUtils.isTrue(DateTools.isCompactDateTime(path))) {
dir = Paths.get(Config.base(), "local", "dump", "dumpData_" + path);
} else {
dir = Paths.get(path);
if ((!Files.exists(dir)) || (!Files.isDirectory(dir))) {
LOGGER.warn("directory not exist: {}.", path);
return false;
throw new IllegalArgumentException("directory not exist: " + path + ".");
} else if (dir.startsWith(Paths.get(Config.base()))) {
LOGGER.warn("path can not in base directory.");
return false;
throw new IllegalArgumentException("path can not in base directory.");
}
}
Thread thread = new Thread(new RunnableImpl(dir, start, classLoader));
thread.start();
return true;
return dir;
}
public class RunnableImpl implements Runnable {
......@@ -101,7 +106,7 @@ public class RestoreData {
@Override
public void run() {
try {
List<String> classNames = entities(catalog);
List<String> classNames = entities(catalog, classLoader);
LOGGER.print("find: {} data to restore, path: {}.", classNames.size(), this.dir.toString());
Path xml = Paths.get(Config.dir_local_temp_classes().getAbsolutePath(),
DateTools.compact(start) + "_restore.xml");
......@@ -118,7 +123,7 @@ public class RestoreData {
Class<JpaObject> cls = (Class<JpaObject>) Thread.currentThread().getContextClassLoader()
.loadClass(className);
LOGGER.print("restore data({}/{}): {}.", idx.getAndAdd(1), classNames.size(), cls.getName());
long size = restore(cls, xml);
long size = restore(cls, Config.dumpRestoreData().getAttachStorage(), xml);
total.getAndAdd(size);
} catch (Exception e) {
LOGGER.error(new Exception(String.format("restore:%s error.", className), e));
......@@ -132,10 +137,7 @@ public class RestoreData {
}
}
private long restore(Class<?> cls, Path xml) throws Exception {
// EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
// xml.getFileName().toString(),
// PersistenceXmlHelper.properties(cls.getName(), Config.slice().getEnable()));
private long restore(Class<?> cls, boolean attachStorage, Path xml) throws Exception {
EntityManagerFactory emf = OpenJPAPersistence.createEntityManagerFactory(cls.getName(),
xml.getFileName().toString(), PersistenceXmlHelper.properties(cls.getName(), false));
EntityManager em = emf.createEntityManager();
......@@ -157,24 +159,7 @@ public class RestoreData {
paths.stream().forEach(o -> {
LOGGER.print("restore {}/{} part of data:{}.", batch.getAndAdd(1), paths.size(), cls.getName());
try {
em.getTransaction().begin();
JsonArray raws = this.convert(o);
for (JsonElement json : raws) {
Object t = gson.fromJson(json, cls);
if (Objects.equals(Config.dumpRestoreData().getRestoreOverride(),
DumpRestoreData.RESTOREOVERRIDE_SKIPEXISTED)
&& (null != em.find(cls, BeanUtils.getBeanProperty(t, JpaObject.id_FIELDNAME)))) {
continue;
}
if (StorageObject.class.isAssignableFrom(cls)) {
Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString()));
this.binary(t, cls, sub, storageMappings);
}
em.persist(t);
count.getAndAdd(1);
}
em.getTransaction().commit();
em.clear();
restore(cls, o, em, attachStorage, storageMappings, count);
} catch (Exception e) {
LOGGER.error(new Exception(String.format("restore error with file:%s.", o.toString()), e));
}
......@@ -189,6 +174,28 @@ public class RestoreData {
return count.longValue();
}
private void restore(Class<?> cls, Path o, EntityManager em, boolean attachStorage,
StorageMappings storageMappings, AtomicLong count) throws Exception {
em.getTransaction().begin();
JsonArray raws = this.convert(o);
for (JsonElement json : raws) {
Object t = gson.fromJson(json, cls);
if (Objects.equals(Config.dumpRestoreData().getRestoreOverride(),
DumpRestoreData.RESTOREOVERRIDE_SKIPEXISTED)
&& (null != em.find(cls, BeanUtils.getBeanProperty(t, JpaObject.id_FIELDNAME)))) {
continue;
}
if (StorageObject.class.isAssignableFrom(cls) && attachStorage) {
Path sub = o.resolveSibling(FilenameUtils.getBaseName(o.getFileName().toString()));
this.binary(t, cls, sub, storageMappings);
}
em.persist(t);
count.getAndAdd(1);
}
em.getTransaction().commit();
em.clear();
}
private List<Path> list(Path directory) throws IOException {
List<Path> list = new ArrayList<>();
try (Stream<Path> stream = Files.list(directory)) {
......@@ -202,21 +209,30 @@ public class RestoreData {
return list;
}
private List<String> entities(DumpRestoreDataCatalog catalog) throws Exception {
List<String> containerEntityNames = new ArrayList<>();
containerEntityNames.addAll(JpaObjectTools.scanContainerEntityNames(classLoader));
List<String> classNames = new ArrayList<>();
classNames.addAll(catalog.keySet());
private List<String> entities(DumpRestoreDataCatalog catalog, ClassLoader classLoader) throws Exception {
List<String> containerEntityNames = new ArrayList<>(JpaObjectTools.scanContainerEntityNames(classLoader));
if (StringUtils.equalsIgnoreCase(DumpRestoreData.MODE_LITE, Config.dumpRestoreData().getMode())) {
containerEntityNames = containerEntityNames.stream().filter(o -> {
try {
ContainerEntity containerEntity = classLoader.loadClass(o).getAnnotation(ContainerEntity.class);
return Objects.equals(containerEntity.reference(), ContainerEntity.Reference.strong);
} catch (Exception e) {
LOGGER.error(e);
}
return false;
}).collect(Collectors.toList());
}
List<String> classNames = new ArrayList<>(catalog.keySet());
classNames = ListTools.includesExcludesWildcard(classNames, Config.dumpRestoreData().getIncludes(),
Config.dumpRestoreData().getExcludes());
return ListTools.includesExcludesWildcard(containerEntityNames, classNames, null);
return ListUtils.intersection(containerEntityNames, classNames);
}
@SuppressWarnings("unchecked")
private void binary(Object o, Class<?> cls, Path sub, StorageMappings storageMappings) throws Exception {
StorageObject so = (StorageObject) o;
StorageMapping mapping = null;
if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistribute())) {
if (BooleanUtils.isTrue(Config.dumpRestoreData().getRedistributeStorage())) {
mapping = storageMappings.random((Class<StorageObject>) cls);
} else {
mapping = storageMappings.get((Class<StorageObject>) cls, so.getStorage());
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或