#12889 Transformer setting. Transform processing

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/pages/database/AttributeTransformerSettingsDialog.java

+/*
+ * DBeaver - Universal Database Manager
+ * Copyright (C) 2010-2021 DBeaver Corp and others
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.jkiss.dbeaver.tools.transfer.ui.pages.database;
+
+import org.eclipse.jface.dialogs.IDialogConstants;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.GridData;
+import org.eclipse.swt.layout.GridLayout;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Label;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.swt.widgets.Text;
+import org.jkiss.dbeaver.Log;
+import org.jkiss.dbeaver.model.preferences.DBPPropertyDescriptor;
+import org.jkiss.dbeaver.runtime.properties.PropertySourceCustom;
+import org.jkiss.dbeaver.tools.transfer.database.DatabaseMappingAttribute;
+import org.jkiss.dbeaver.tools.transfer.registry.DataTransferAttributeTransformerDescriptor;
+import org.jkiss.dbeaver.ui.UIUtils;
+import org.jkiss.dbeaver.ui.dialogs.BaseDialog;
+import org.jkiss.dbeaver.ui.properties.PropertyTreeViewer;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+class AttributeTransformerSettingsDialog extends BaseDialog {
+
+    private static final Log log = Log.getLog(AttributeTransformerSettingsDialog.class);
+
+    private final DatabaseMappingAttribute mapping;
+    private final DataTransferAttributeTransformerDescriptor transformer;
+
+    private PropertyTreeViewer propertiesEditor;
+    private PropertySourceCustom propertySource;
+
+    private Text infoText;
+
+    public AttributeTransformerSettingsDialog(Shell parentShell, DatabaseMappingAttribute mapping, DataTransferAttributeTransformerDescriptor transformer) {
+        super(parentShell, "Transformer " + transformer.getName() + " settings", null);
+        this.mapping = mapping;
+        this.transformer = transformer;
+    }
+
+    @Override
+    protected Composite createDialogArea(Composite parent)
+    {
+        Composite composite = super.createDialogArea(parent);
+
+        createTransformSettingsArea(composite);
+
+        updateTransformerInfo();
+
+        return parent;
+    }
+
+    private void updateTransformerInfo() {
+        if (infoText != null) {
+            if (transformer != null && transformer.getDescription() != null) {
+                infoText.setText(transformer.getDescription());
+            } else {
+                infoText.setText("");
+            }
+        }
+
+        if (transformer != null) {
+            Collection<? extends DBPPropertyDescriptor> transformerProperties = transformer.getProperties();
+            loadTransformerSettings(transformerProperties);
+        } else {
+            loadTransformerSettings(Collections.emptyList());
+        }
+    }
+
+    private void saveTransformerSettings() {
+        propertiesEditor.saveEditorValues();
+        Map<String, Object> settings = mapping.getTransformerProperties();
+        if (settings == null) {
+            settings = new LinkedHashMap<>();
+        }
+        final Map<String, Object> properties = propertySource.getPropertiesWithDefaults();
+        for (Map.Entry<String, Object> prop : properties.entrySet()) {
+            if (prop.getValue() != null) {
+                settings.put(prop.getKey(), prop.getValue());
+            }
+        }
+        mapping.setTransformerProperties(settings);
+    }
+
+    private void createTransformSettingsArea(Composite composite) {
+        Composite settingsPanel = UIUtils.createComposite(composite, 1);
+        if (composite.getLayout() instanceof GridLayout) {
+            settingsPanel.setLayoutData(new GridData(GridData.FILL_BOTH));
+        }
+        final Composite placeholder = UIUtils.createComposite(settingsPanel, 2);
+        UIUtils.createLabelText(placeholder, "Transformer", transformer.getName(), SWT.READ_ONLY);
+        Label infoLabel = UIUtils.createControlLabel(placeholder, "Info");
+        infoLabel.setLayoutData(new GridData(GridData.VERTICAL_ALIGN_BEGINNING));
+        infoText = new Text(placeholder, SWT.READ_ONLY | SWT.WRAP);
+        GridData gd = new GridData(GridData.FILL_HORIZONTAL);
+        gd.widthHint = 300;
+        infoText.setLayoutData(gd);
+
+        propertiesEditor = new PropertyTreeViewer(settingsPanel, SWT.BORDER);
+
+        propertiesEditor.getControl().setFocus();
+    }
+
+    private void loadTransformerSettings(Collection<? extends DBPPropertyDescriptor> properties) {
+        Map<String, Object> transformOptions = mapping.getTransformerProperties();
+        if (transformOptions == null) {
+            transformOptions = Collections.emptyMap();
+        }
+        propertySource = new PropertySourceCustom(
+            properties,
+            transformOptions);
+        propertiesEditor.loadProperties(propertySource);
+    }
+
+    @Override
+    protected void createButtonsForButtonBar(Composite parent)
+    {
+		createButton(parent, IDialogConstants.OK_ID, IDialogConstants.OK_LABEL, true);
+        createButton(parent, IDialogConstants.CANCEL_ID, IDialogConstants.CANCEL_LABEL, false);
+    }
+
+    @Override
+    public void create() {
+        super.create();
+
+        if (propertySource != null && propertySource.getProperties().length == 0) {
+            // No properties
+            UIUtils.asyncExec(this::okPressed);
+        }
+    }
+
+    @Override
+    protected void okPressed()
+    {
+        saveTransformerSettings();
+
+        super.okPressed();
+    }
+
+    @Override
+    public boolean close() {
+        return super.close();
+    }
+}

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/pages/database/DatabaseConsumerPageMapping.java

@@ -16,6 +16,7 @@
  */
 package org.jkiss.dbeaver.tools.transfer.ui.pages.database;
 
+import org.eclipse.jface.action.Action;
 import org.eclipse.jface.dialogs.IDialogConstants;
 import org.eclipse.jface.dialogs.IMessageProvider;
 import org.eclipse.jface.viewers.*;
@@ -429,6 +430,31 @@ public class DatabaseConsumerPageMapping extends ActiveWizardPage<DataTransferWi
         mappingViewer.getTree().setLinesVisible(true);
         mappingViewer.getTree().setHeaderVisible(true);
 
+        UIUtils.setControlContextMenu(mappingViewer.getTree(), manager -> {
+            IStructuredSelection selection = (IStructuredSelection) mappingViewer.getSelection();
+            if (!selection.isEmpty()) {
+                Object element = selection.getFirstElement();
+                if (element instanceof DatabaseMappingAttribute) {
+                    DatabaseMappingAttribute mapping= (DatabaseMappingAttribute) element;
+                    if (mapping.getTransformer() != null && !mapping.getTransformer().getProperties().isEmpty()) {
+                        manager.add(new Action("Transformer settings ...") {
+                            @Override
+                            public void run() {
+                                AttributeTransformerSettingsDialog settingsDialog = new AttributeTransformerSettingsDialog(
+                                    getShell(),
+                                    (DatabaseMappingAttribute) element,
+                                    mapping.getTransformer());
+                                if (settingsDialog.open() != IDialogConstants.OK_ID) {
+                                    return;
+                                }
+                            }
+                        });
+                    }
+                }
+            }
+            UIUtils.fillDefaultTreeContextMenu(manager, mappingViewer.getTree());
+        });
+
         {
             TreeViewerColumn columnSource = new TreeViewerColumn(mappingViewer, SWT.LEFT);
             columnSource.setLabelProvider(new MappingLabelProvider() {
@@ -566,7 +592,8 @@ public class DatabaseConsumerPageMapping extends ActiveWizardPage<DataTransferWi
 
                 @Override
                 protected Object getValue(Object element) {
-                    return getTransformer(element);
+                    DataTransferAttributeTransformerDescriptor transformer = getTransformer(element);
+                    return transformer == null ? "" : transformer.getName();
                 }
 
                 @Override
@@ -579,9 +606,18 @@ public class DatabaseConsumerPageMapping extends ActiveWizardPage<DataTransferWi
                         newTransformer = DataTransferRegistry.getInstance().getAttributeTransformerByName(tName);
                     }
                     if (element instanceof DatabaseMappingAttribute) {
+                        if (newTransformer != null && !newTransformer.getProperties().isEmpty()) {
+                            AttributeTransformerSettingsDialog settingsDialog = new AttributeTransformerSettingsDialog(
+                                getShell(),
+                                (DatabaseMappingAttribute) element,
+                                newTransformer);
+                            if (settingsDialog.open() != IDialogConstants.OK_ID) {
+                                return;
+                            }
+                        }
                         ((DatabaseMappingAttribute) element).setTransformer(newTransformer);
+                        mappingViewer.refresh();
                     }
-                    mappingViewer.refresh();
                     setErrorMessage(null);
                 }
             });

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/DataTransferJob.java

@@ -33,14 +33,14 @@ import java.util.Locale;
  */
 public class DataTransferJob implements DBRRunnableWithProgress {
 
-    private DataTransferSettings settings;
-    private DBTTask task;
+    private final DataTransferSettings settings;
+    private final DBTTask task;
     private long elapsedTime;
     private boolean hasErrors;
 
-    private Locale locale;
-    private Log log;
-    private DBTTaskExecutionListener listener;
+    private final Locale locale;
+    private final Log log;
+    private final DBTTaskExecutionListener listener;
 
     public DataTransferJob(DataTransferSettings settings, DBTTask task, Locale locale, Log log, DBTTaskExecutionListener listener)
     {
@@ -64,7 +64,7 @@ public class DataTransferJob implements DBRRunnableWithProgress {
     }
 
     @Override
-    public void run(DBRProgressMonitor monitor) throws InvocationTargetException, InterruptedException {
+    public void run(DBRProgressMonitor monitor) throws InvocationTargetException {
         monitor.beginTask("Perform data transfer", 1);
         hasErrors = false;
         long startTime = System.currentTimeMillis();

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/IDataTransferAttributeTransformer.java

@@ -17,6 +17,7 @@
 package org.jkiss.dbeaver.tools.transfer;
 
 import org.jkiss.code.NotNull;
+import org.jkiss.code.Nullable;
 import org.jkiss.dbeaver.DBException;
 import org.jkiss.dbeaver.model.data.DBDAttributeBinding;
 import org.jkiss.dbeaver.model.exec.DBCSession;
@@ -33,6 +34,7 @@ public interface IDataTransferAttributeTransformer {
         @NotNull DBDAttributeBinding[] dataAttributes,
         @NotNull Object[] dataRow,
         @NotNull DBDAttributeBinding attribute,
+        @Nullable Object attrValue,
         @NotNull Map<String, Object> options)
         throws DBException;
 

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/database/DatabaseMappingAttribute.java

@@ -25,6 +25,7 @@ import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
 import org.jkiss.dbeaver.model.runtime.VoidProgressMonitor;
 import org.jkiss.dbeaver.model.struct.*;
 import org.jkiss.dbeaver.tools.transfer.registry.DataTransferAttributeTransformerDescriptor;
+import org.jkiss.dbeaver.tools.transfer.registry.DataTransferRegistry;
 import org.jkiss.dbeaver.tools.transfer.stream.StreamDataImporterColumnInfo;
 import org.jkiss.utils.CommonUtils;
 
@@ -277,7 +278,7 @@ public class DatabaseMappingAttribute implements DatabaseMappingObject {
 
             if (transformer != null) {
                 settings.put("transformer", transformer.getId());
-                settings.put("properties", new LinkedHashMap<>(transformerProperties));
+                settings.put("transformerProperties", new LinkedHashMap<>(transformerProperties));
             }
         }
     }
@@ -309,6 +310,19 @@ public class DatabaseMappingAttribute implements DatabaseMappingObject {
                 log.error(e);
             }
         }
+        Object transformerId = settings.get("transformer");
+        if (transformerId != null) {
+            transformer = DataTransferRegistry.getInstance().getAttributeTransformer(transformerId.toString());
+            if (transformer == null) {
+                log.error("Can't find attribute transformer " + transformerId);
+            } else {
+                Map<String, Object> tp = (Map<String, Object>) settings.get("transformerProperties");
+                transformerProperties.clear();
+                if (tp != null) {
+                    transformerProperties.putAll(tp);
+                }
+            }
+        }
     }
 
     @Override

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/database/DatabaseTransferConsumer.java

@@ -40,6 +40,7 @@ import org.jkiss.dbeaver.model.struct.rdb.DBSManipulationType;
 import org.jkiss.dbeaver.model.struct.rdb.DBSSchema;
 import org.jkiss.dbeaver.runtime.DBWorkbench;
 import org.jkiss.dbeaver.runtime.ui.DBPPlatformUI;
+import org.jkiss.dbeaver.tools.transfer.IDataTransferAttributeTransformer;
 import org.jkiss.dbeaver.tools.transfer.IDataTransferConsumer;
 import org.jkiss.dbeaver.tools.transfer.IDataTransferNodePrimary;
 import org.jkiss.dbeaver.tools.transfer.IDataTransferProcessor;
@@ -78,6 +79,7 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
 
     private boolean isPreview;
     private List<Object[]> previewRows;
+    private DBDAttributeBinding[] rsAttributes;
 
     public static class ColumnMapping {
         public DBDAttributeBinding sourceAttr;
@@ -85,6 +87,8 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
         public DBDValueHandler sourceValueHandler;
         public DBDValueHandler targetValueHandler;
         public int targetIndex = -1;
+        public IDataTransferAttributeTransformer valueTransformer;
+        public Map<String, Object> valueTransformerProperties;
 
         private ColumnMapping(DBDAttributeBinding sourceAttr) {
             this.sourceAttr = sourceAttr;
@@ -160,7 +164,6 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
             }
         }
 
-        DBDAttributeBinding[] rsAttributes;
         boolean dynamicTarget = targetContext.getDataSource().getInfo().isDynamicMetadata();
         DBSDataContainer sourceObject = getSourceObject();
         if (dynamicTarget) {
@@ -216,6 +219,14 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
             if (columnMapping.targetAttr.getMappingType() == DatabaseMappingType.skip) {
                 continue;
             }
+            if (columnMapping.targetAttr.getTransformer() != null) {
+                try {
+                    columnMapping.valueTransformer = columnMapping.targetAttr.getTransformer().createTransformer();
+                    columnMapping.valueTransformerProperties = columnMapping.targetAttr.getTransformerProperties();
+                } catch (DBException e) {
+                    throw new DBCException("Can't create attribute transformer", e);
+                }
+            }
             DBSEntityAttribute targetAttr = columnMapping.targetAttr.getTarget();
             if (targetAttr == null) {
                 if (isPreview) {
@@ -266,7 +277,7 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
             if (column == null || column.targetIndex < 0) {
                 continue;
             }
-            final Object attrValue;
+            Object attrValue;
             if (column.sourceValueHandler != null) {
                 if (column.sourceAttr instanceof DBDAttributeBindingCustom) {
                     attrValue = DBUtils.getAttributeValue(column.sourceAttr, sourceBindings, rowValues);
@@ -277,6 +288,7 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
                 // No value handler - get raw value
                 attrValue = resultSet.getAttributeValue(i);
             }
+
             if (containerMapping != null && containerMapping.getTarget() instanceof DBSDocumentContainer) {
                 rowValues[column.targetIndex] = attrValue;
             } else {
@@ -288,6 +300,31 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
                     false, false);
             }
         }
+
+        // Transform value
+        for (ColumnMapping column : columnMappings) {
+            if (column == null || column.targetIndex < 0) {
+                continue;
+            }
+            if (column.valueTransformer != null) {
+                Object attrValue = rowValues[column.targetIndex];
+                try {
+                    rowValues[column.targetIndex] = column.valueTransformer.transformAttribute(
+                        session,
+                        rsAttributes,
+                        rowValues,
+                        column.sourceAttr,
+                        attrValue,
+                        column.valueTransformerProperties);
+                } catch (DBException e) {
+                    throw new DBCException(
+                        "Error transforming attribute '" + column.sourceAttr.getName() +
+                            "' value with transformer '" + column.targetAttr.getTransformer().getName() + "'", e);
+                }
+            }
+        }
+
+
         executeBatch.add(rowValues);
 
         rowsExported++;

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/registry/DataTransferAttributeTransformerDescriptor.java

@@ -20,11 +20,13 @@ package org.jkiss.dbeaver.tools.transfer.registry;
 import org.eclipse.core.runtime.IConfigurationElement;
 import org.jkiss.code.NotNull;
 import org.jkiss.code.Nullable;
+import org.jkiss.dbeaver.DBException;
 import org.jkiss.dbeaver.model.DBIcon;
 import org.jkiss.dbeaver.model.DBPImage;
 import org.jkiss.dbeaver.model.impl.AbstractDescriptor;
 import org.jkiss.dbeaver.model.impl.PropertyDescriptor;
 import org.jkiss.dbeaver.model.preferences.DBPPropertyDescriptor;
+import org.jkiss.dbeaver.tools.transfer.IDataTransferAttributeTransformer;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -40,6 +42,7 @@ public class DataTransferAttributeTransformerDescriptor extends AbstractDescript
     private final String description;
     @NotNull
     private final DBPImage icon;
+    private final ObjectType implType;
     private final List<DBPPropertyDescriptor> properties = new ArrayList<>();
 
     public DataTransferAttributeTransformerDescriptor(IConfigurationElement config) {
@@ -49,6 +52,7 @@ public class DataTransferAttributeTransformerDescriptor extends AbstractDescript
         this.name = config.getAttribute("label");
         this.description = config.getAttribute("description");
         this.icon = iconToImage(config.getAttribute("icon"), DBIcon.TYPE_UNKNOWN);
+        this.implType = new ObjectType(config.getAttribute("class"));
 
         for (IConfigurationElement prop : config.getChildren(PropertyDescriptor.TAG_PROPERTY_GROUP)) {
             properties.addAll(PropertyDescriptor.extractProperties(prop));
@@ -80,6 +84,16 @@ public class DataTransferAttributeTransformerDescriptor extends AbstractDescript
         return properties;
     }
 
+    public IDataTransferAttributeTransformer createTransformer() throws DBException
+    {
+        try {
+            return implType.getObjectClass(IDataTransferAttributeTransformer.class)
+                .getDeclaredConstructor().newInstance();
+        } catch (Throwable e) {
+            throw new DBException("Can't create attribute transformer instance", e);
+        }
+    }
+
     @Override
     public String toString() {
         return id;

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/registry/DataTransferRegistry.java

@@ -174,6 +174,6 @@ public class DataTransferRegistry {
     }
 
     public DataTransferAttributeTransformerDescriptor getAttributeTransformerByName(String tName) {
-        return transformers.values().stream().filter(t -> t.getName().equals(tName)).findFirst().get();
+        return transformers.values().stream().filter(t -> t.getName().equals(tName)).findFirst().orElse(null);
     }
 }

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/transformers/DataTransferTransformerConstant.java

@@ -30,7 +30,7 @@ import java.util.Map;
 public class DataTransferTransformerConstant implements IDataTransferAttributeTransformer {
 
     @Override
-    public Object transformAttribute(@NotNull DBCSession session, @NotNull DBDAttributeBinding[] dataAttributes, @NotNull Object[] dataRow, @NotNull DBDAttributeBinding attribute, @NotNull Map<String, Object> options) throws DBException {
+    public Object transformAttribute(@NotNull DBCSession session, @NotNull DBDAttributeBinding[] dataAttributes, @NotNull Object[] dataRow, @NotNull DBDAttributeBinding attribute, Object attrValue, @NotNull Map<String, Object> options) throws DBException {
         return null;
     }
 

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/transformers/DataTransferTransformerExpression.java

@@ -30,7 +30,7 @@ import java.util.Map;
 public class DataTransferTransformerExpression implements IDataTransferAttributeTransformer {
 
     @Override
-    public Object transformAttribute(@NotNull DBCSession session, @NotNull DBDAttributeBinding[] dataAttributes, @NotNull Object[] dataRow, @NotNull DBDAttributeBinding attribute, @NotNull Map<String, Object> options) throws DBException {
+    public Object transformAttribute(@NotNull DBCSession session, @NotNull DBDAttributeBinding[] dataAttributes, @NotNull Object[] dataRow, @NotNull DBDAttributeBinding attribute, Object attrValue, @NotNull Map<String, Object> options) throws DBException {
         return null;
     }
 

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/transformers/DataTransferTransformerNull.java

@@ -30,7 +30,7 @@ import java.util.Map;
 public class DataTransferTransformerNull implements IDataTransferAttributeTransformer {
 
     @Override
-    public Object transformAttribute(@NotNull DBCSession session, @NotNull DBDAttributeBinding[] dataAttributes, @NotNull Object[] dataRow, @NotNull DBDAttributeBinding attribute, @NotNull Map<String, Object> options) throws DBException {
+    public Object transformAttribute(@NotNull DBCSession session, @NotNull DBDAttributeBinding[] dataAttributes, @NotNull Object[] dataRow, @NotNull DBDAttributeBinding attribute, Object attrValue, @NotNull Map<String, Object> options) throws DBException {
         return null;
     }