#6909 Stream producer settings save/load fix

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/pages/database/DatabaseConsumerPageLoadSettings.java

@@ -37,7 +37,6 @@ public class DatabaseConsumerPageLoadSettings extends ActiveWizardPage<DataTrans
         super("Data load");
         setTitle("Data load settings");
         setDescription("Configuration of table data load");
-        setPageComplete(false);
     }
 
     @Override

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/pages/database/DatabaseConsumerPageMapping.java

@@ -82,7 +82,6 @@ public class DatabaseConsumerPageMapping extends ActiveWizardPage<DataTransferWi
         super("Tables mapping");
         setTitle("Tables mapping");
         setDescription("Map tables and columns transfer");
-        setPageComplete(false);
     }
 
     private DatabaseConsumerSettings getDatabaseConsumerSettings() {

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/pages/stream/StreamProducerPagePreview.java

@@ -38,16 +38,18 @@ import org.jkiss.dbeaver.model.exec.DBCSession;
 import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
 import org.jkiss.dbeaver.model.runtime.DefaultProgressMonitor;
 import org.jkiss.dbeaver.model.struct.DBSEntity;
-import org.jkiss.dbeaver.model.struct.DBSEntityAttribute;
 import org.jkiss.dbeaver.model.struct.DBSObject;
 import org.jkiss.dbeaver.runtime.DBWorkbench;
+import org.jkiss.dbeaver.tools.transfer.DataTransferPipe;
 import org.jkiss.dbeaver.tools.transfer.IDataTransferConsumer;
 import org.jkiss.dbeaver.tools.transfer.IDataTransferProcessor;
 import org.jkiss.dbeaver.tools.transfer.IDataTransferSettings;
 import org.jkiss.dbeaver.tools.transfer.internal.DTMessages;
 import org.jkiss.dbeaver.tools.transfer.registry.DataTransferProcessorDescriptor;
-import org.jkiss.dbeaver.tools.transfer.stream.*;
-import org.jkiss.dbeaver.tools.transfer.DataTransferPipe;
+import org.jkiss.dbeaver.tools.transfer.stream.IStreamDataImporter;
+import org.jkiss.dbeaver.tools.transfer.stream.StreamDataImporterColumnInfo;
+import org.jkiss.dbeaver.tools.transfer.stream.StreamProducerSettings;
+import org.jkiss.dbeaver.tools.transfer.stream.StreamTransferProducer;
 import org.jkiss.dbeaver.tools.transfer.ui.wizard.DataTransferWizard;
 import org.jkiss.dbeaver.ui.DBeaverIcons;
 import org.jkiss.dbeaver.ui.UIIcon;
@@ -56,10 +58,6 @@ import org.jkiss.dbeaver.ui.controls.CustomTableEditor;
 import org.jkiss.dbeaver.ui.dialogs.ActiveWizardPage;
 import org.jkiss.utils.CommonUtils;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.List;
@@ -80,7 +78,6 @@ public class StreamProducerPagePreview extends ActiveWizardPage<DataTransferWiza
         super(DTMessages.data_transfer_wizard_page_preview_name);
         setTitle(DTMessages.data_transfer_wizard_page_preview_title);
         setDescription(DTMessages.data_transfer_wizard_page_preview_description);
-        setPageComplete(false);
     }
 
     private StreamProducerSettings getProducerSettings() {
@@ -275,6 +272,8 @@ public class StreamProducerPagePreview extends ActiveWizardPage<DataTransferWiza
         }
 
         setControl(composite);
+
+        determinePageCompletion();
     }
 
     @Override
@@ -350,21 +349,9 @@ public class StreamProducerPagePreview extends ActiveWizardPage<DataTransferWiza
                 DBRProgressMonitor monitor = new DefaultProgressMonitor(mon);
                 monitor.beginTask("Load mappings", 3);
                 try {
-                    monitor.subTask("Load attributes form target object");
-                    for (DBSEntityAttribute attr : CommonUtils.safeCollection(entity.getAttributes(monitor))) {
-                        if (DBUtils.isPseudoAttribute(attr) || DBUtils.isHiddenObject(attr)) {
-                            continue;
-                        }
-                        entityMapping.getAttributeMapping(attr);
-                    }
+                    getProducerSettings().updateMappingsFromStream(getWizard().getSettings());
                     monitor.worked(1);
 
-                    // Load header and mappings
-                    monitor.subTask("Load attribute mappings");
-                    if (importer instanceof IStreamDataImporter) {
-                        loadStreamMappings((IStreamDataImporter)importer, entity, currentProducer);
-                    }
-
                     UIUtils.syncExec(() -> updateAttributeMappings(entityMapping));
                     monitor.worked(1);
 
@@ -386,6 +373,8 @@ public class StreamProducerPagePreview extends ActiveWizardPage<DataTransferWiza
         } catch (InterruptedException e) {
             // Ignore
         }
+        determinePageCompletion();
+
         Throwable finalError = error;
         UIUtils.asyncExec(() -> {
             UIUtils.packColumns(mappingsTable, true);
@@ -395,6 +384,7 @@ public class StreamProducerPagePreview extends ActiveWizardPage<DataTransferWiza
                 DBWorkbench.getPlatformUI().showError("Load entity meta", "Can't load entity attributes", finalError);
             }
         });
+
     }
 
     private void refreshPreviewTable(StreamProducerSettings.EntityMapping entityMapping) {
@@ -495,58 +485,6 @@ public class StreamProducerPagePreview extends ActiveWizardPage<DataTransferWiza
         return pipeList.get(tableList.getSelectionIndex());
     }
 
-    private void loadStreamMappings(IStreamDataImporter importer, DBSEntity entity, StreamTransferProducer currentProducer) throws DBException {
-        File inputFile = currentProducer.getInputFile();
-
-        final StreamProducerSettings settings = getProducerSettings();
-        final Map<Object, Object> processorProperties = getWizard().getSettings().getProcessorProperties();
-        StreamProducerSettings.EntityMapping entityMapping = settings.getEntityMapping(entity);
-
-        List<StreamDataImporterColumnInfo> columnInfos;
-        try (InputStream is = new FileInputStream(inputFile)) {
-            importer.init(new StreamDataImporterSite(settings, entity, processorProperties));
-            columnInfos = importer.readColumnsInfo(is);
-            importer.dispose();
-        } catch (IOException e) {
-            throw new DBException("IO error", e);
-        }
-        entityMapping.setStreamColumns(columnInfos);
-
-        // Map source columns
-        List<StreamProducerSettings.AttributeMapping> attributeMappings = entityMapping.getAttributeMappings();
-        for (StreamDataImporterColumnInfo columnInfo : columnInfos) {
-            boolean mappingFound = false;
-            if (columnInfo.getColumnName() != null) {
-                for (StreamProducerSettings.AttributeMapping attr : attributeMappings) {
-                    if (CommonUtils.equalObjects(attr.getTargetAttributeName(), columnInfo.getColumnName())) {
-                        if (attr.getMappingType() == StreamProducerSettings.AttributeMapping.MappingType.NONE) {
-                            // Set source name only if it wasn't set
-                            attr.setSourceAttributeName(columnInfo.getColumnName());
-                            attr.setSourceAttributeIndex(columnInfo.getColumnIndex());
-                            attr.setMappingType(StreamProducerSettings.AttributeMapping.MappingType.IMPORT);
-                            attr.setSourceColumn(columnInfo);
-                        }
-                        mappingFound = true;
-                        break;
-                    }
-                }
-            }
-            if (!mappingFound) {
-                if (columnInfo.getColumnIndex() >= 0 && columnInfo.getColumnIndex() < attributeMappings.size()) {
-                    StreamProducerSettings.AttributeMapping attr = attributeMappings.get(columnInfo.getColumnIndex());
-                    if (attr.getMappingType() == StreamProducerSettings.AttributeMapping.MappingType.NONE) {
-                        if (!CommonUtils.isEmpty(columnInfo.getColumnName())) {
-                            attr.setSourceAttributeName(columnInfo.getColumnName());
-                        }
-                        attr.setSourceAttributeIndex(columnInfo.getColumnIndex());
-                        attr.setMappingType(StreamProducerSettings.AttributeMapping.MappingType.IMPORT);
-                        attr.setSourceColumn(columnInfo);
-                    }
-                }
-            }
-        }
-    }
-
     private void loadImportPreview(DBRProgressMonitor monitor, IStreamDataImporter importer, DBSEntity entity, StreamTransferProducer currentProducer) throws DBException {
         final StreamProducerSettings settings = getProducerSettings();
 

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/pages/stream/StreamProducerPageSettings.java

@@ -54,7 +54,6 @@ public class StreamProducerPageSettings extends ActiveWizardPage<DataTransferWiz
         super(DTMessages.data_transfer_wizard_page_input_files_name);
         setTitle(DTMessages.data_transfer_wizard_page_input_files_title);
         setDescription(DTMessages.data_transfer_wizard_page_input_files_description);
-        setPageComplete(false);
     }
 
     @Override
@@ -106,6 +105,8 @@ public class StreamProducerPageSettings extends ActiveWizardPage<DataTransferWiz
         settingsDivider.setWeights(new int[]{ 400, 600 });
 
         setControl(settingsDivider);
+
+        updatePageCompletion();
     }
 
     private boolean chooseSourceFile(DataTransferPipe pipe) {

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/wizard/DataTransferPageFinal.java

@@ -53,7 +53,6 @@ class DataTransferPageFinal extends ActiveWizardPage<DataTransferWizard> {
         super(DTMessages.data_transfer_wizard_final_name);
         setTitle(DTMessages.data_transfer_wizard_final_title);
         setDescription(DTMessages.data_transfer_wizard_final_description);
-        setPageComplete(false);
     }
 
     @Override

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/wizard/DataTransferPagePipes.java

@@ -64,7 +64,6 @@ class DataTransferPagePipes extends ActiveWizardPage<DataTransferWizard> {
 
     DataTransferPagePipes() {
         super(DTMessages.data_transfer_wizard_init_name);
-        setPageComplete(false);
     }
 
     @Override
@@ -84,6 +83,8 @@ class DataTransferPagePipes extends ActiveWizardPage<DataTransferWizard> {
         updatePageCompletion();
 
         setControl(composite);
+
+        updatePageCompletion();
     }
 
     private void createNodesTable(Composite composite) {

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/wizard/DataTransferPageSettings.java

@@ -41,7 +41,6 @@ class DataTransferPageSettings extends ActiveWizardPage<DataTransferWizard> {
         super(DTMessages.data_transfer_wizard_settings_name);
         setTitle(DTMessages.data_transfer_wizard_settings_title);
         setDescription(DTMessages.data_transfer_wizard_settings_description);
-        setPageComplete(false);
     }
 
     @Override

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/stream/StreamProducerSettings.java

@@ -16,16 +16,31 @@
  */
 package org.jkiss.dbeaver.tools.transfer.stream;
 
+import org.jkiss.dbeaver.DBException;
+import org.jkiss.dbeaver.Log;
 import org.jkiss.dbeaver.model.DBPEvaluationContext;
 import org.jkiss.dbeaver.model.DBUtils;
+import org.jkiss.dbeaver.model.app.DBPProject;
 import org.jkiss.dbeaver.model.data.DBDValueHandler;
+import org.jkiss.dbeaver.model.data.json.JSONUtils;
+import org.jkiss.dbeaver.model.exec.DBCException;
+import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
 import org.jkiss.dbeaver.model.runtime.DBRRunnableContext;
 import org.jkiss.dbeaver.model.struct.DBSEntity;
 import org.jkiss.dbeaver.model.struct.DBSEntityAttribute;
+import org.jkiss.dbeaver.model.struct.DBSObject;
+import org.jkiss.dbeaver.runtime.DBWorkbench;
+import org.jkiss.dbeaver.tools.transfer.DataTransferPipe;
 import org.jkiss.dbeaver.tools.transfer.DataTransferSettings;
+import org.jkiss.dbeaver.tools.transfer.IDataTransferProcessor;
 import org.jkiss.dbeaver.tools.transfer.IDataTransferSettings;
 import org.jkiss.utils.CommonUtils;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
 import java.util.*;
 
 /**
@@ -33,21 +48,49 @@ import java.util.*;
  */
 public class StreamProducerSettings implements IDataTransferSettings {
 
+    private static final Log log = Log.getLog(StreamProducerSettings.class);
+
     public static class EntityMapping {
         private String entityName;
         private DBSEntity entity;
         private List<AttributeMapping> attributeMappings = new ArrayList<>();
         private List<StreamDataImporterColumnInfo> streamColumns = new ArrayList<>();
 
-        public EntityMapping(DBSEntity entity) {
+        EntityMapping(DBSEntity entity) {
             this.entity = entity;
             this.entityName = DBUtils.getObjectFullName(entity, DBPEvaluationContext.DML);
         }
 
-        public EntityMapping(String entityName) {
+        EntityMapping(String entityName) {
             this.entityName = entityName;
         }
 
+        EntityMapping(DBRRunnableContext runnableContext, Map<String, Object> config) throws DBCException, InvocationTargetException, InterruptedException {
+            String projectName = CommonUtils.toString(config.get("project"));
+            DBPProject project = CommonUtils.isEmpty(projectName) ? null : DBWorkbench.getPlatform().getWorkspace().getProject(projectName);
+            if (project == null) {
+                throw new DBCException("Error loading stream producer mappings - project not found");
+            }
+
+            String entityId = CommonUtils.toString(config.get("entityId"));
+            if (!CommonUtils.isEmpty(entityId)) {
+                runnableContext.run(true, true, monitor -> {
+                    try {
+                        entity = (DBSEntity) DBUtils.findObjectById(monitor, project, entityId);
+
+                        Map<String, Object> attrsConfig = JSONUtils.getObject(config, "attributes");
+                        for (String sourceAttrName : attrsConfig.keySet()) {
+                            Map<String, Object> attrMap = (Map<String, Object>) attrsConfig.get(sourceAttrName);
+                            AttributeMapping am = new AttributeMapping(monitor, EntityMapping.this, sourceAttrName, attrMap);
+                            attributeMappings.add(am);
+                        }
+                    } catch (DBException e) {
+                        throw new InvocationTargetException(e);
+                    }
+                });
+            }
+        }
+
         public String getEntityName() {
             return entityName;
         }
@@ -114,6 +157,7 @@ public class StreamProducerSettings implements IDataTransferSettings {
             }
             Map<String, Object> mappings = new LinkedHashMap<>();
 
+            mappings.put("project", entity.getDataSource().getContainer().getProject().getName());
             mappings.put("entityId", DBUtils.getObjectFullId(entity));
             Map<String, Object> attrConfig = new LinkedHashMap<>();
             mappings.put("attributes", attrConfig);
@@ -126,8 +170,6 @@ public class StreamProducerSettings implements IDataTransferSettings {
 
     public static class AttributeMapping {
 
-        private final DBDValueHandler targetValueHandler;
-
         public enum MappingType {
             NONE("none"),
             IMPORT("import"),
@@ -146,6 +188,8 @@ public class StreamProducerSettings implements IDataTransferSettings {
         }
 
         private DBSEntityAttribute targetAttribute;
+        private DBDValueHandler targetValueHandler;
+
         private String targetAttributeName;
         private String sourceAttributeName;
         private int sourceAttributeIndex = -1;
@@ -155,11 +199,24 @@ public class StreamProducerSettings implements IDataTransferSettings {
         private StreamDataImporterColumnInfo sourceColumn;
 
         public AttributeMapping(DBSEntityAttribute attr) {
-            this.targetAttribute = attr;
             this.targetAttributeName = attr.getName();
+            this.targetAttribute = attr;
             this.targetValueHandler = DBUtils.findValueHandler(attr.getDataSource(), attr);
         }
 
+        AttributeMapping(DBRProgressMonitor monitor, EntityMapping em, String sourceAttrName, Map<String, Object> config) throws DBException {
+            this.targetAttributeName = JSONUtils.getString(config, "targetName");
+            this.sourceAttributeName = sourceAttrName;
+            this.mappingType = CommonUtils.valueOf(MappingType.class, CommonUtils.toString(config.get("mapping")), MappingType.SKIP);
+            this.skip = JSONUtils.getBoolean(config, "skip");
+            this.defaultValue = JSONUtils.getString(config, "default");
+
+            this.targetAttribute = em.entity.getAttribute(monitor, this.targetAttributeName);
+            if (this.targetAttribute != null) {
+                this.targetValueHandler = DBUtils.findValueHandler(this.targetAttribute.getDataSource(), this.targetAttribute);
+            }
+        }
+
         public MappingType getMappingType() {
             return mappingType;
         }
@@ -283,6 +340,92 @@ public class StreamProducerSettings implements IDataTransferSettings {
     @Override
     public void loadSettings(DBRRunnableContext runnableContext, DataTransferSettings dataTransferSettings, Map<String, Object> settings) {
         setProcessorProperties(dataTransferSettings.getProcessorProperties());
+
+        try {
+            for (Map<String, Object> mapping : JSONUtils.getObjectList(settings, "mappings")) {
+                EntityMapping em = new EntityMapping(runnableContext, mapping);
+                DBSEntity entity = em.getEntity();
+                if (entity != null) {
+                    entityMapping.put(DBUtils.getObjectFullName(entity, DBPEvaluationContext.DML), em);
+                }
+            }
+            updateMappingsFromStream(dataTransferSettings);
+        } catch (Exception e) {
+            log.error("Error loading stream producer settings", e);
+        }
+    }
+
+    public void updateMappingsFromStream(DataTransferSettings dataTransferSettings) throws DBException {
+        final Map<Object, Object> processorProperties = dataTransferSettings.getProcessorProperties();
+        for (DataTransferPipe pipe : dataTransferSettings.getDataPipes()) {
+            StreamTransferProducer producer = (StreamTransferProducer) pipe.getProducer();
+            if (producer != null && pipe.getConsumer() != null) {
+                DBSObject targetObject = pipe.getConsumer().getDatabaseObject();
+                if (targetObject instanceof DBSEntity) {
+                    DBSEntity entity = (DBSEntity) targetObject;
+
+                    StreamProducerSettings.EntityMapping entityMapping = getEntityMapping(entity);
+
+                    List<StreamDataImporterColumnInfo> columnInfos = null;
+                    File inputFile = producer.getInputFile();
+
+                    IDataTransferProcessor importer = dataTransferSettings.getProcessor().getInstance();
+
+                    if (importer instanceof IStreamDataImporter) {
+                        IStreamDataImporter sdi = (IStreamDataImporter)importer;
+                        try (InputStream is = new FileInputStream(inputFile)) {
+                            sdi.init(new StreamDataImporterSite(this, entity, processorProperties));
+                            try {
+                                columnInfos = sdi.readColumnsInfo(is);
+                            } finally {
+                                sdi.dispose();
+                            }
+                        } catch (IOException e) {
+                            throw new DBException("IO error", e);
+                        }
+                    }
+                    entityMapping.setStreamColumns(columnInfos);
+
+                    // Map source columns
+                    if (columnInfos != null) {
+                        List<StreamProducerSettings.AttributeMapping> attributeMappings = entityMapping.getAttributeMappings();
+                        for (StreamDataImporterColumnInfo columnInfo : columnInfos) {
+                            boolean mappingFound = false;
+                            if (columnInfo.getColumnName() != null) {
+                                for (StreamProducerSettings.AttributeMapping attr : attributeMappings) {
+                                    if (CommonUtils.equalObjects(attr.getTargetAttributeName(), columnInfo.getColumnName())) {
+                                        if (attr.getMappingType() == StreamProducerSettings.AttributeMapping.MappingType.NONE || attr.getSourceAttributeIndex() < 0) {
+                                            // Set source name only if it wasn't set
+                                            attr.setSourceAttributeName(columnInfo.getColumnName());
+                                            attr.setSourceAttributeIndex(columnInfo.getColumnIndex());
+                                            if (attr.getMappingType() == StreamProducerSettings.AttributeMapping.MappingType.NONE) {
+                                                attr.setMappingType(StreamProducerSettings.AttributeMapping.MappingType.IMPORT);
+                                            }
+                                            attr.setSourceColumn(columnInfo);
+                                        }
+                                        mappingFound = true;
+                                        break;
+                                    }
+                                }
+                            }
+                            if (!mappingFound) {
+                                if (columnInfo.getColumnIndex() >= 0 && columnInfo.getColumnIndex() < attributeMappings.size()) {
+                                    StreamProducerSettings.AttributeMapping attr = attributeMappings.get(columnInfo.getColumnIndex());
+                                    if (attr.getMappingType() == StreamProducerSettings.AttributeMapping.MappingType.NONE) {
+                                        if (!CommonUtils.isEmpty(columnInfo.getColumnName())) {
+                                            attr.setSourceAttributeName(columnInfo.getColumnName());
+                                        }
+                                        attr.setSourceAttributeIndex(columnInfo.getColumnIndex());
+                                        attr.setMappingType(StreamProducerSettings.AttributeMapping.MappingType.IMPORT);
+                                        attr.setSourceColumn(columnInfo);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
     }
 
     @Override

plugins/org.jkiss.dbeaver.model/src/org/jkiss/dbeaver/model/data/json/JSONUtils.java

@@ -305,13 +305,12 @@ public class JSONUtils {
     }
 
     @NotNull
-    public static Collection<Map<String, Object>> getObjectList(@NotNull Map<String, Object> map, @NotNull String name) {
-        List<Map<String, Object>> list = (List<Map<String, Object>> ) map.get(name);
-        if (list == null) {
-            return Collections.emptyList();
-        } else {
-            return list;
+    public static List<Map<String, Object>> getObjectList(@NotNull Map<String, Object> map, @NotNull String name) {
+        Object value = map.get(name);
+        if (value instanceof List) {
+            return  (List<Map<String, Object>>) value;
         }
+        return Collections.emptyList();
     }
 
     @Nullable

plugins/org.jkiss.dbeaver.ui.navigator/src/org/jkiss/dbeaver/ui/task/TaskConfigurationWizardPageTask.java

@@ -74,7 +74,6 @@ class TaskConfigurationWizardPageTask extends ActiveWizardPage
         super(task == null ? "Create new task" : "Edit task");
         setTitle(task == null ? "New task properties" : "Edit task properties");
         setDescription("Set task name, type and input data");
-        setPageComplete(false);
 
         this.task = (TaskImpl) task;
         if (this.task != null) {