#8994 PG copy bulk load + data load settings UI fixes

plugins/org.jkiss.dbeaver.data.transfer.ui/src/org/jkiss/dbeaver/tools/transfer/ui/pages/database/DatabaseConsumerPageLoadSettings.java

@@ -59,7 +59,7 @@ public class DatabaseConsumerPageLoadSettings extends DataTransferPageNodeSettin
     private Group loadSettings;
     private String disableReferentialIntegrityCheckboxTooltip;
     private boolean isDisablingReferentialIntegritySupported;
-    private Spinner multiRowInsertBatch;
+    private Text multiRowInsertBatch;
     private Button skipBindValues;
     private Button useBatchCheck;
 
@@ -125,7 +125,7 @@ public class DatabaseConsumerPageLoadSettings extends DataTransferPageNodeSettin
 
             UIUtils.createControlLabel(loadSettings, DTUIMessages.database_consumer_wizard_on_duplicate_key_insert_method_text);
             onDuplicateKeyInsertMethods = new Combo(loadSettings, SWT.DROP_DOWN | SWT.READ_ONLY);
-            onDuplicateKeyInsertMethods.setLayoutData(new GridData(GridData.HORIZONTAL_ALIGN_BEGINNING));
+            onDuplicateKeyInsertMethods.setLayoutData(new GridData(GridData.FILL_HORIZONTAL));
 
             Link urlLabel = UIUtils.createLink(loadSettings, "<a href=\"" + HelpUtils.getHelpExternalReference(HELP_TOPIC_REPLACE_METHOD) + "\">"
                     + DTUIMessages.database_consumer_wizard_link_label_replace_method_wiki + "</a>", new SelectionAdapter() {
@@ -179,16 +179,18 @@ public class DatabaseConsumerPageLoadSettings extends DataTransferPageNodeSettin
                 }
             });
 
-            final Spinner commitAfterEdit = UIUtils.createLabelSpinner(performanceSettings, DTUIMessages.database_consumer_wizard_commit_spinner_label, settings.getCommitAfterRows(), 1, Integer.MAX_VALUE);
+            final Text commitAfterEdit = UIUtils.createLabelText(performanceSettings, DTUIMessages.database_consumer_wizard_commit_spinner_label, String.valueOf(settings.getCommitAfterRows()), SWT.BORDER);
             commitAfterEdit.addSelectionListener(new SelectionAdapter() {
                 @Override
                 public void widgetSelected(SelectionEvent e) {
-                    settings.setCommitAfterRows(commitAfterEdit.getSelection());
+                    settings.setCommitAfterRows(CommonUtils.toInt(commitAfterEdit.getText()));
                 }
             });
-            commitAfterEdit.setLayoutData(new GridData(GridData.HORIZONTAL_ALIGN_BEGINNING, GridData.VERTICAL_ALIGN_BEGINNING, false, false, 3, 1));
+            GridData gd = new GridData(GridData.HORIZONTAL_ALIGN_BEGINNING, GridData.VERTICAL_ALIGN_BEGINNING, false, false, 3, 1);
+            gd.widthHint = UIUtils.getFontHeight(commitAfterEdit) * 6;
+            commitAfterEdit.setLayoutData(gd);
 
-            final Button useMultiRowInsert = UIUtils.createCheckbox(performanceSettings, DTUIMessages.database_consumer_wizard_checkbox_multi_insert_label, DTUIMessages.database_consumer_wizard_checkbox_multi_insert_description, settings.isUseMultiRowInsert(), 4);
+            final Button useMultiRowInsert = UIUtils.createCheckbox(performanceSettings, DTUIMessages.database_consumer_wizard_checkbox_multi_insert_label, DTUIMessages.database_consumer_wizard_checkbox_multi_insert_description, settings.isUseMultiRowInsert(), 1);
             if (useBatchCheck != null && ((!useBatchCheck.isDisposed() && useBatchCheck.getSelection())
             || (useBatchCheck.isDisposed() && settings.isDisableUsingBatches()))) {
                 useMultiRowInsert.setEnabled(false);
@@ -207,17 +209,21 @@ public class DatabaseConsumerPageLoadSettings extends DataTransferPageNodeSettin
                 }
             });
 
-            multiRowInsertBatch = UIUtils.createLabelSpinner(performanceSettings, DTUIMessages.database_consumer_wizard_spinner_multi_insert_batch_size, settings.getMultiRowInsertBatch(), 2, Integer.MAX_VALUE);
+            multiRowInsertBatch = new Text(performanceSettings, SWT.BORDER);
+            multiRowInsertBatch.setToolTipText(DTUIMessages.database_consumer_wizard_spinner_multi_insert_batch_size);
+            gd = new GridData(GridData.FILL_HORIZONTAL);
+            gd.horizontalSpan = 3;
+            multiRowInsertBatch.setLayoutData(gd);
+            multiRowInsertBatch.setText(String.valueOf(settings.getMultiRowInsertBatch()));
             if (!useMultiRowInsert.getSelection() || useBatchCheck != null && !useBatchCheck.isDisposed() && useBatchCheck.getSelection()) {
                 multiRowInsertBatch.setEnabled(false);
             }
             multiRowInsertBatch.addSelectionListener(new SelectionAdapter() {
                 @Override
                 public void widgetSelected(SelectionEvent e) {
-                    settings.setMultiRowInsertBatch(multiRowInsertBatch.getSelection());
+                    settings.setMultiRowInsertBatch(CommonUtils.toInt(multiRowInsertBatch.getText()));
                 }
             });
-            multiRowInsertBatch.setLayoutData(new GridData(GridData.HORIZONTAL_ALIGN_BEGINNING, GridData.VERTICAL_ALIGN_BEGINNING, false, false, 3, 1));
 
             skipBindValues = UIUtils.createCheckbox(performanceSettings, DTUIMessages.database_consumer_wizard_checkbox_multi_insert_skip_bind_values_label, DTUIMessages.database_consumer_wizard_checkbox_multi_insert_skip_bind_values_description, settings.isSkipBindValues(), 4);
             skipBindValues.addSelectionListener(new SelectionAdapter() {
@@ -314,6 +320,9 @@ public class DatabaseConsumerPageLoadSettings extends DataTransferPageNodeSettin
 
     private boolean confirmDataTruncate() {
         Shell shell = getContainer().getShell();
+        if (shell == null) {
+            return true;
+        }
         if (shell.isVisible() || getSettings().isTruncateBeforeLoad()) {
             String tableNames = getWizard().getSettings().getDataPipes().stream().map(pipe -> pipe.getConsumer() == null ? "" : pipe.getConsumer().getObjectName()).collect(Collectors.joining(","));
             String checkbox_question = NLS.bind(DTUIMessages.database_consumer_wizard_truncate_checkbox_question, tableNames);
@@ -339,6 +348,7 @@ public class DatabaseConsumerPageLoadSettings extends DataTransferPageNodeSettin
             insertMethodsDescriptors = dialectDescriptor.getSupportedInsertReplaceMethodsDescriptors();
         }
 
+        onDuplicateKeyInsertMethods.removeAll();
         onDuplicateKeyInsertMethods.add(DBSDataManipulator.INSERT_NONE_METHOD);
         if (!CommonUtils.isEmpty(insertMethodsDescriptors)) {
             boolean emptyButton = true;

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/database/DatabaseConsumerSettings.java

@@ -58,7 +58,7 @@ public class DatabaseConsumerSettings implements IDataTransferSettings {
     private boolean truncateBeforeLoad = false;
     private boolean openTableOnFinish = true;
     private boolean useMultiRowInsert;
-    private int multiRowInsertBatch = 100;
+    private int multiRowInsertBatch = 500;
     private boolean skipBindValues;
     private boolean disableUsingBatches = false;
     private String onDuplicateKeyInsertMethodId;

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/database/DatabaseTransferConsumer.java

@@ -366,6 +366,7 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
             if (needCommit) {
                 bulkLoadManager.flushRows(targetSession);
             }
+            return;
         } else {
             boolean disableUsingBatches = settings.isDisableUsingBatches();
             if ((needCommit || disableUsingBatches) && executeBatch != null) {
@@ -452,12 +453,7 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
                 insertBatch(true);
             }
             if (bulkLoadManager != null) {
-                try {
-                    bulkLoadManager.finishBulkLoad(targetSession);
-                } finally {
-                    bulkLoadManager.close();
-                }
-                bulkLoadManager = null;
+                bulkLoadManager.finishBulkLoad(targetSession);
             } else if (executeBatch != null) {
                 executeBatch.close();
                 executeBatch = null;
@@ -514,7 +510,7 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
         }
     }
 
-    DBSObject checkTargetContainer(DBRProgressMonitor monitor) throws DBException {
+    private DBSObject checkTargetContainer(DBRProgressMonitor monitor) throws DBException {
         DBSDataManipulator targetObject = getTargetObject();
         if (targetObject == null) {
             if (settings.getContainerNode() != null && settings.getContainerNode().getDataSource() == null) {
@@ -553,6 +549,11 @@ public class DatabaseTransferConsumer implements IDataTransferConsumer<DatabaseC
             targetContext.close();
             targetContext = null;
         }
+
+        if (bulkLoadManager != null) {
+            bulkLoadManager.close();
+            bulkLoadManager = null;
+        }
     }
 
     @Override

plugins/org.jkiss.dbeaver.data.transfer/src/org/jkiss/dbeaver/tools/transfer/stream/importer/DataImporterCSV.java

@@ -19,6 +19,7 @@ package org.jkiss.dbeaver.tools.transfer.stream.importer;
 import org.jkiss.code.NotNull;
 import org.jkiss.dbeaver.DBException;
 import org.jkiss.dbeaver.Log;
+import org.jkiss.dbeaver.model.DBFetchProgress;
 import org.jkiss.dbeaver.model.DBPDataKind;
 import org.jkiss.dbeaver.model.DBPDataSource;
 import org.jkiss.dbeaver.model.DBUtils;
@@ -270,8 +271,8 @@ public class DataImporterCSV extends StreamImporterAbstract {
                         consumer.fetchRow(producerSession, resultSet);
                         lineNum++;
 
-                        if (lineNum % 1000 == 0) {
-                            monitor.subTask(String.valueOf(lineNum) + " rows processed");
+                        if (DBFetchProgress.monitorFetchProgress(lineNum)) {
+                            monitor.subTask(lineNum + " rows processed");
                         }
                     }
                 }

plugins/org.jkiss.dbeaver.ext.greenplum/src/org/jkiss/dbeaver/ext/greenplum/model/PostgreServerGreenplum.java

@@ -143,6 +143,11 @@ public class PostgreServerGreenplum extends PostgreServerExtensionBase {
         return true;
     }
 
+    @Override
+    public boolean supportsCopyFromStdIn() {
+        return true;
+    }
+
     @Override
     public boolean supportsExternalTypes() {
         return true;

plugins/org.jkiss.dbeaver.ext.postgresql/src/org/jkiss/dbeaver/ext/postgresql/model/PostgreCopyLoader.java

@@ -168,8 +168,11 @@ public class PostgreCopyLoader implements DBSDataBulkLoader, DBSDataBulkLoader.B
         }
         csvWriter = null;
 
-        String queryText = "COPY " + table.getFullyQualifiedName(DBPEvaluationContext.DML) +
-            " FROM STDIN (FORMAT csv)";
+        String tableFQN = table.getFullyQualifiedName(DBPEvaluationContext.DML);
+
+        session.getProgressMonitor().subTask("Copy into " + tableFQN);
+
+        String queryText = "COPY " + tableFQN + " FROM STDIN (FORMAT csv)";
 
         try {
             Reader csvReader = new FileReader(csvFile, StandardCharsets.UTF_8);
@@ -178,6 +181,7 @@ public class PostgreCopyLoader implements DBSDataBulkLoader, DBSDataBulkLoader.B
             // Commit changes
             DBCTransactionManager txnManager = DBUtils.getTransactionManager(session.getExecutionContext());
             if (txnManager != null && !txnManager.isAutoCommit()) {
+                session.getProgressMonitor().subTask("Commit COPY");
                 txnManager.commit(session);
             }
 
@@ -191,6 +195,7 @@ public class PostgreCopyLoader implements DBSDataBulkLoader, DBSDataBulkLoader.B
     }
 
     @Override
+
     public void close() {
         if (csvFile != null && csvFile.exists()) {
             if (!csvFile.delete()) {

plugins/org.jkiss.dbeaver.ext.postgresql/src/org/jkiss/dbeaver/ext/postgresql/model/PostgreServerExtension.java

@@ -152,7 +152,10 @@ public interface PostgreServerExtension {
 
     boolean supportsAlterUserChangePassword();
 
+    boolean supportsCopyFromStdIn();
+
     int getParameterBindType(DBSTypedObject type, Object value);
 
     int getTruncateToolModes();
+
 }

plugins/org.jkiss.dbeaver.ext.postgresql/src/org/jkiss/dbeaver/ext/postgresql/model/PostgreTableReal.java

@@ -274,7 +274,9 @@ public abstract class PostgreTableReal extends PostgreTableBase implements DBPOb
     @Override
     public <T> T getAdapter(Class<T> adapter) {
         if (adapter == DBSDataBulkLoader.class) {
-            return adapter.cast(new PostgreCopyLoader(this));
+            if (getDataSource().getServerType().supportsCopyFromStdIn()) {
+                return adapter.cast(new PostgreCopyLoader(this));
+            }
         }
         return null;
     }

plugins/org.jkiss.dbeaver.ext.postgresql/src/org/jkiss/dbeaver/ext/postgresql/model/impls/PostgreServerCockroachDB.java

@@ -252,6 +252,11 @@ public class PostgreServerCockroachDB extends PostgreServerExtensionBase {
         return true;
     }
 
+    @Override
+    public boolean supportsCopyFromStdIn() {
+        return true;
+    }
+
     @Override
     public int getTruncateToolModes() {
         return TRUNCATE_TOOL_MODE_SUPPORT_ONLY_ONE_TABLE | TRUNCATE_TOOL_MODE_SUPPORT_CASCADE;

plugins/org.jkiss.dbeaver.ext.postgresql/src/org/jkiss/dbeaver/ext/postgresql/model/impls/PostgreServerEdb.java

@@ -34,6 +34,11 @@ public class PostgreServerEdb extends PostgreServerExtensionBase {
         return true;
     }
 
+    @Override
+    public boolean supportsCopyFromStdIn() {
+        return true;
+    }
+
     @Override
     public String getServerTypeName() {
         return "EnterpriseDB";

plugins/org.jkiss.dbeaver.ext.postgresql/src/org/jkiss/dbeaver/ext/postgresql/model/impls/PostgreServerExtensionBase.java

@@ -495,6 +495,11 @@ public abstract class PostgreServerExtensionBase implements PostgreServerExtensi
         return false;
     }
 
+    @Override
+    public boolean supportsCopyFromStdIn() {
+        return false;
+    }
+
     @Override
     public int getParameterBindType(DBSTypedObject type, Object value) {
         return Types.OTHER;

plugins/org.jkiss.dbeaver.ext.postgresql/src/org/jkiss/dbeaver/ext/postgresql/model/impls/PostgreServerPostgreSQL.java

@@ -80,4 +80,9 @@ public class PostgreServerPostgreSQL extends PostgreServerExtensionBase {
     public boolean supportsAlterUserChangePassword() {
         return true;
     }
+
+    @Override
+    public boolean supportsCopyFromStdIn() {
+        return true;
+    }
 }

plugins/org.jkiss.dbeaver.ext.postgresql/src/org/jkiss/dbeaver/ext/postgresql/model/impls/PostgreServerTimescale.java

@@ -32,6 +32,11 @@ public class PostgreServerTimescale extends PostgreServerExtensionBase {
         return true;
     }
 
+    @Override
+    public boolean supportsCopyFromStdIn() {
+        return true;
+    }
+
     @Override
     public String getServerTypeName() {
         return "Timescale";

plugins/org.jkiss.dbeaver.model/src/org/jkiss/dbeaver/model/DBFetchProgress.java

@@ -70,8 +70,10 @@ public final class DBFetchProgress {
             return fetchedRows % 100 == 0;
         } else if (fetchedRows < 100000) {
             return fetchedRows % 1000 == 0;
-        } else {
+        } else if (fetchedRows < 1000000) {
             return fetchedRows % 10000 == 0;
+        } else {
+            return fetchedRows % 100000 == 0;
         }
     }