Merge pull request #11962 from dbeaver/sql-completion-analyzer-use-rules#10858

#10858 Eliminate the usage of regex in favor of dialect-specific rules

Former-commit-id: 516a2199

bundles/org.jkiss.utils/src/org/jkiss/utils/CommonUtils.java

@@ -743,7 +743,7 @@ public class CommonUtils {
         return String.format("%1$"+length+ "s", string);
     }
 
-    public static boolean startsWithIgnoreCase(@NotNull String str, @NotNull String startPart) {
+    public static boolean startsWithIgnoreCase(@Nullable String str, @Nullable String startPart) {
         if (isEmpty(str) || isEmpty(startPart)) {
             return false;
         }

plugins/org.jkiss.dbeaver.model.sql/src/org/jkiss/dbeaver/model/sql/completion/SQLCompletionAnalyzer.java

@@ -19,6 +19,8 @@ package org.jkiss.dbeaver.model.sql.completion;
 import net.sf.jsqlparser.schema.Table;
 import net.sf.jsqlparser.statement.Statement;
 import net.sf.jsqlparser.util.TablesNamesFinder;
+import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.jface.text.IDocument;
 import org.jkiss.code.NotNull;
 import org.jkiss.code.Nullable;
 import org.jkiss.dbeaver.DBException;
@@ -38,17 +40,20 @@ import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor;
 import org.jkiss.dbeaver.model.runtime.DBRRunnableParametrized;
 import org.jkiss.dbeaver.model.sql.*;
 import org.jkiss.dbeaver.model.sql.parser.SQLParserPartitions;
+import org.jkiss.dbeaver.model.sql.parser.SQLRuleManager;
 import org.jkiss.dbeaver.model.sql.parser.SQLWordPartDetector;
+import org.jkiss.dbeaver.model.sql.parser.tokens.SQLTokenType;
 import org.jkiss.dbeaver.model.struct.*;
 import org.jkiss.dbeaver.model.text.TextUtils;
+import org.jkiss.dbeaver.model.text.parser.TPRuleBasedScanner;
+import org.jkiss.dbeaver.model.text.parser.TPToken;
+import org.jkiss.dbeaver.model.text.parser.TPTokenDefault;
 import org.jkiss.utils.ArrayUtils;
 import org.jkiss.utils.CommonUtils;
+import org.jkiss.utils.Pair;
 
 import java.lang.reflect.InvocationTargetException;
 import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
 
 /**
  * Completion analyzer
@@ -168,7 +173,7 @@ public class SQLCompletionAnalyzer implements DBRRunnableParametrized<DBRProgres
                 DBPObject rootObject = null;
                 if (queryType == SQLCompletionRequest.QueryType.COLUMN && dataSource instanceof DBSObjectContainer) {
                     // Try to detect current table
-                    rootObject = getTableFromAlias((DBSObjectContainer)dataSource, null, true);
+                    rootObject = getTableFromAlias((DBSObjectContainer)dataSource, null);
                     if (rootObject instanceof DBSEntity) {
                         switch (prevKeyWord) {
                             case SQLConstants.KEYWORD_ON:
@@ -210,7 +215,7 @@ public class SQLCompletionAnalyzer implements DBRRunnableParametrized<DBRProgres
                 }
                 if (queryType == SQLCompletionRequest.QueryType.JOIN && !proposals.isEmpty() && dataSource instanceof DBSObjectContainer) {
                     // Filter out non-joinable tables
-                    DBSObject leftTable = getTableFromAlias((DBSObjectContainer) dataSource, null, true);
+                    DBSObject leftTable = getTableFromAlias((DBSObjectContainer) dataSource, null);
                     if (leftTable instanceof DBSEntity) {
                         filterNonJoinableProposals((DBSEntity)leftTable);
                     }
@@ -244,14 +249,14 @@ public class SQLCompletionAnalyzer implements DBRRunnableParametrized<DBRProgres
                     }
                     if (tableAlias == null && !CommonUtils.isEmpty(wordPart)) {
                         // May be an incomplete table alias. Try to find such table
-                        rootObject = getTableFromAlias(sc, wordPart, false);
+                        rootObject = getTableFromAlias(sc, wordPart);
                         if (rootObject != null) {
                             // Found alias - no proposals
                             searchFinished = true;
                             return;
                         }
                     }
-                    rootObject = getTableFromAlias(sc, tableAlias, false);
+                    rootObject = getTableFromAlias(sc, tableAlias);
                     if (rootObject == null && tableAlias != null) {
                         // Maybe alias ss a table name
                         String[] allNames = SQLUtils.splitFullIdentifier(
@@ -540,19 +545,10 @@ public class SQLCompletionAnalyzer implements DBRRunnableParametrized<DBRProgres
             if (wordPart.indexOf(request.getContext().getSyntaxManager().getStructSeparator()) != -1 || wordPart.equals(ALL_COLUMNS_PATTERN)) {
                 return;
             }
-            SQLDialect sqlDialect = request.getContext().getDataSource().getSQLDialect();
-            String tableNamePattern = getTableNamePattern(sqlDialect);
-            String tableAliasPattern = getTableAliasPattern("(" + wordPart + "[a-z]*)", tableNamePattern);
-            Pattern rp = Pattern.compile(tableAliasPattern);
-            // Append trailing space to let alias regex match correctly
-            Matcher matcher = rp.matcher(request.getActiveQuery().getText() + " ");
-            while (matcher.find()) {
-                String tableName = matcher.group(1);
-                String tableAlias = matcher.group(2);
-                if (tableAlias.equals(wordPart)) {
-                    continue;
-                }
-
+            final Pair<String, String> name = extractTableName(wordPart, true);
+            if (name != null) {
+                final String tableName = name.getFirst();
+                final String tableAlias = name.getSecond();
                 if (!hasProposal(proposals, tableName)) {
                     proposals.add(
                         0,
@@ -716,7 +712,7 @@ public class SQLCompletionAnalyzer implements DBRRunnableParametrized<DBRProgres
             if (childObject == null) {
                 if (i == 0) {
                     // Assume it's a table alias ?
-                    childObject = getTableFromAlias(sc, token, false);
+                    childObject = getTableFromAlias(sc, token);
                     if (childObject == null && !request.isSimpleMode()) {
                         // Search using structure assistant
                         DBSStructureAssistant structureAssistant = DBUtils.getAdapter(DBSStructureAssistant.class, sc);
@@ -784,7 +780,7 @@ public class SQLCompletionAnalyzer implements DBRRunnableParametrized<DBRProgres
     }
 
     @Nullable
-    private DBSObject getTableFromAlias(DBSObjectContainer sc, @Nullable String token, boolean firstMatch)
+    private DBSObject getTableFromAlias(DBSObjectContainer sc, @Nullable String token)
     {
         if (token == null) {
             token = "";
@@ -796,90 +792,111 @@ public class SQLCompletionAnalyzer implements DBRRunnableParametrized<DBRProgres
         if (dataSource == null) {
             return null;
         }
-        SQLScriptElement activeQuery = request.getActiveQuery();
-        if (activeQuery == null) {
-            return null;
-        }
 
-        final List<String> nameList = new ArrayList<>();
-        SQLDialect sqlDialect = dataSource.getSQLDialect();
-        {
-            // Regex matching MUST be very fast.
-            // Otherwise UI will freeze during SQL typing.
-            // So let's make regex as simple as possible.
-            // TODO: will be replaced by SQL preparse + structure analysis
-
-//            String quote = quoteString == null ? SQLConstants.STR_QUOTE_DOUBLE :
-//                SQLConstants.STR_QUOTE_DOUBLE.equals(quoteString) || SQLConstants.STR_QUOTE_APOS.equals(quoteString) ?
-//                    quoteString :
-//                    Pattern.quote(quoteString);
-            String catalogSeparator = sqlDialect.getCatalogSeparator();
+        final SQLDialect sqlDialect = dataSource.getSQLDialect();
+        final String catalogSeparator = sqlDialect.getCatalogSeparator();
+
         while (token.endsWith(catalogSeparator)) {
-                token = token.substring(0, token.length() -1);
+            token = token.substring(0, token.length() - 1);
         }
 
-            // Use silly pattern with all possible characters
-            // Valid regex for quote identifiers and FQ names is monstrous and very slow
-            String tableNamePattern = getTableNamePattern(sqlDialect);
-            String structNamePattern;
-            if (CommonUtils.isEmpty(token)) {
-                String kwList = "from|update|join|into";
-                if (request.getQueryType() != SQLCompletionRequest.QueryType.COLUMN) {
-                    kwList = kwList + "|,";
-                }
-                structNamePattern = "(?:" + kwList + ")\\s+" + tableNamePattern;
-            } else {
-                structNamePattern = getTableAliasPattern(token, tableNamePattern);
+        final Pair<String, String> name = extractTableName(token, false);
+        if (name != null && CommonUtils.isNotEmpty(name.getFirst())) {
+            final String[][] quoteStrings = sqlDialect.getIdentifierQuoteStrings();
+            final String[] allNames = SQLUtils.splitFullIdentifier(name.getFirst(), catalogSeparator, quoteStrings, false);
+            return SQLSearchUtils.findObjectByFQN(monitor, sc, request.getContext().getExecutionContext(), Arrays.asList(allNames), !request.isSimpleMode(), request.getWordDetector());
         }
 
-            Pattern aliasPattern;
-            try {
-                aliasPattern = Pattern.compile(structNamePattern, Pattern.CASE_INSENSITIVE);
-            } catch (PatternSyntaxException e) {
-                // Bad pattern - seems to be a bad token
         return null;
     }
-            // Append trailing space to let alias regex match correctly
-            String testQuery = SQLUtils.stripComments(request.getContext().getSyntaxManager().getDialect(), activeQuery.getText()) + " ";
-            Matcher matcher = aliasPattern.matcher(testQuery);
-            while (matcher.find()) {
-                if (!nameList.isEmpty() && (firstMatch || matcher.start() > request.getDocumentOffset() - activeQuery.getOffset())) {
-                    // Do not search after cursor
+
+    @Nullable
+    private Pair<String, String> extractTableName(@Nullable String tableAlias, boolean allowPartialMatch) {
+        final IDocument document = request.getDocument();
+        final SQLScriptElement activeQuery = request.getActiveQuery();
+        final SQLRuleManager ruleManager = request.getContext().getRuleManager();
+        final TPRuleBasedScanner scanner = new TPRuleBasedScanner();
+        scanner.setRules(ruleManager.getAllRules());
+        scanner.setRange(document, activeQuery.getOffset(), activeQuery.getLength());
+
+         /*
+            When we search for table name knowing its alias, we want to match the following sequence:
+                [FROM|UPDATE|JOIN|INTO] <table-name> [AS]? <known-alias-name>
+
+            If we don't know the alias, the following sequence must be used instead:
+                [FROM|UPDATE|JOIN|INTO] <table-name>
+
+            We use "state machine" to process such sequences. The transition table is listed below:
+                UNMATCHED  -> TABLE_NAME ; if found starting token (FROM, UPDATE, JOIN, INTO, etc.).
+                TABLE_NAME -> ALIAS_AS   ; if found string token, and the alias is known.
+                TABLE_NAME -> MATCHED    ; if found string token, and the alias is unknown.
+                ALIAS_AS   -> ALIAS_NAME ; if found 'as' token.
+                ALIAS_NAME -> MATCHED    ; if found string token.
+         */
+
+        try {
+            final int STATE_UNMATCHED   = 0;
+            final int STATE_TABLE_NAME  = 1;
+            final int STATE_ALIAS_AS    = 2;
+            final int STATE_ALIAS_NAME  = 3;
+            final int STATE_MATCHED     = 4;
+
+            int state = STATE_UNMATCHED;
+            String matchedTableName = null;
+            String matchedTableAlias = null;
+
+            while (true) {
+                final TPToken tok = scanner.nextToken();
+                if (tok.isEOF()) {
                     break;
                 }
-                nameList.clear();
-                int groupCount = matcher.groupCount();
-                for (int i = 1; i <= groupCount; i++) {
-                    String group = matcher.group(i);
-                    if (!CommonUtils.isEmpty(group)) {
-                        String[][] quoteStrings = sqlDialect.getIdentifierQuoteStrings();
-
-                        String[] allNames = SQLUtils.splitFullIdentifier(group, catalogSeparator, quoteStrings, false);
-                        Collections.addAll(nameList, allNames);
+                if (!(tok instanceof TPTokenDefault)) {
+                    continue;
+                }
+                final String value = document.get(scanner.getTokenOffset(), scanner.getTokenLength());
+                if (state == STATE_UNMATCHED && tok.getData() == SQLTokenType.T_KEYWORD &&
+                    (value.equalsIgnoreCase(SQLConstants.KEYWORD_FROM) ||
+                        value.equalsIgnoreCase(SQLConstants.KEYWORD_UPDATE) ||
+                        value.equalsIgnoreCase(SQLConstants.KEYWORD_JOIN) ||
+                        value.equalsIgnoreCase(SQLConstants.KEYWORD_INTO))) {
+                    state = STATE_TABLE_NAME;
+                    continue;
                 }
+                if (state == STATE_TABLE_NAME && (tok.getData() == SQLTokenType.T_QUOTED || tok.getData() == SQLTokenType.T_OTHER)) {
+                    matchedTableName = value;
+                    if (CommonUtils.isEmpty(tableAlias)) {
+                        state = STATE_MATCHED;
+                    } else {
+                        state = STATE_ALIAS_AS;
+                        continue;
                     }
                 }
+                if (state == STATE_ALIAS_AS && tok.getData() == SQLTokenType.T_KEYWORD && "AS".equalsIgnoreCase(value)) {
+                    state = STATE_ALIAS_NAME;
+                    continue;
                 }
-
-        return SQLSearchUtils.findObjectByFQN(monitor, sc, request.getContext().getExecutionContext(), nameList, !request.isSimpleMode(), request.getWordDetector());
+                if ((state == STATE_ALIAS_AS || state == STATE_ALIAS_NAME) && (tok.getData() == SQLTokenType.T_QUOTED || tok.getData() == SQLTokenType.T_OTHER)) {
+                    matchedTableAlias = value;
+                    state = STATE_MATCHED;
+                }
+                if (state == STATE_MATCHED) {
+                    final boolean fullMatch = CommonUtils.isEmpty(tableAlias) || tableAlias.equals(matchedTableAlias);
+                    final boolean partialMatch = fullMatch || (allowPartialMatch && CommonUtils.startsWithIgnoreCase(matchedTableAlias, tableAlias));
+                    if (!fullMatch && !partialMatch) {
+                        // The presented alias does not fully or partially match the matched token, reset
+                        state = STATE_UNMATCHED;
+                        matchedTableName = null;
+                        matchedTableAlias = null;
+                    } else {
+                        return new Pair<>(matchedTableName, matchedTableAlias);
                     }
-
-    private String getTableAliasPattern(String alias, String tableNamePattern) {
-        return tableNamePattern + "\\s+(?:as\\s)?" + alias + "[\\s,]+";
                 }
-
-    private static String getTableNamePattern(SQLDialect sqlDialect) {
-        String[][] quoteStrings = sqlDialect.getIdentifierQuoteStrings();
-        StringBuilder quotes = new StringBuilder();
-        if (quoteStrings != null) {
-            for (String[] quotePair : quoteStrings) {
-                if (quotes.indexOf(quotePair[0]) == -1) quotes.append('\\').append(quotePair[0]);
-                if (quotes.indexOf(quotePair[1]) == -1) quotes.append('\\').append(quotePair[1]);
             }
+        } catch (BadLocationException e) {
+            log.debug(e);
         }
-        // Use silly pattern with all possible characters
-        // Valid regex for quote identifiers and FQ names is monstrous and very slow
-        return "([\\p{L}0-9_$§#@\\.\\-" + quotes.toString() + "]+)";
+
+        return null;
     }
 
     private void makeProposalsFromChildren(DBPObject parent, @Nullable String startPart, boolean addFirst, Map<String, Object> params) throws DBException {
@@ -1115,7 +1132,7 @@ public class SQLCompletionAnalyzer implements DBRRunnableParametrized<DBRProgres
                             if (aliases.contains(s) || sqlDialect.getKeywordType(s) != null) {
                                 return true;
                             }
-                            return Pattern.compile("\\s+" + s + "[^\\w]+").matcher(queryText).find();
+                            return extractTableName(s, false) != null;
                         });
                         if (alias.equalsIgnoreCase(object.getName())) {
                             // Don't use alias, when it's identical to entity name

plugins/org.jkiss.dbeaver.model.sql/src/org/jkiss/dbeaver/model/sql/completion/SQLCompletionContext.java

@@ -24,6 +24,7 @@ import org.jkiss.dbeaver.model.DBPKeywordType;
 import org.jkiss.dbeaver.model.DBPNamedObject;
 import org.jkiss.dbeaver.model.exec.DBCExecutionContext;
 import org.jkiss.dbeaver.model.sql.SQLSyntaxManager;
+import org.jkiss.dbeaver.model.sql.parser.SQLRuleManager;
 
 import java.util.Map;
 
@@ -42,6 +43,8 @@ public interface SQLCompletionContext {
 
     SQLSyntaxManager getSyntaxManager();
 
+    SQLRuleManager getRuleManager();
+
     boolean isUseFQNames();
 
     boolean isReplaceWords();

plugins/org.jkiss.dbeaver.ui.editors.sql/src/org/jkiss/dbeaver/ui/editors/sql/SQLEditorBase.java

@@ -251,6 +251,14 @@ public abstract class SQLEditorBase extends BaseTextEditor implements DBPContext
         return syntaxManager;
     }
 
+    @Nullable
+    public SQLRuleManager getRuleManager() {
+        if (parserContext == null) {
+            return null;
+        }
+        return parserContext.getRuleManager();
+    }
+
     @NotNull
     public SQLRuleScanner getRuleScanner() {
         return ruleScanner;

plugins/org.jkiss.dbeaver.ui.editors.sql/src/org/jkiss/dbeaver/ui/editors/sql/syntax/SQLEditorCompletionContext.java

@@ -28,6 +28,7 @@ import org.jkiss.dbeaver.model.sql.SQLSyntaxManager;
 import org.jkiss.dbeaver.model.sql.completion.SQLCompletionContext;
 import org.jkiss.dbeaver.model.sql.completion.SQLCompletionProposalBase;
 import org.jkiss.dbeaver.model.sql.completion.SQLCompletionRequest;
+import org.jkiss.dbeaver.model.sql.parser.SQLRuleManager;
 import org.jkiss.dbeaver.ui.editors.sql.SQLEditorBase;
 import org.jkiss.dbeaver.ui.editors.sql.SQLPreferenceConstants;
 
@@ -59,6 +60,11 @@ public class SQLEditorCompletionContext implements SQLCompletionContext
         return editor.getSyntaxManager();
     }
 
+    @Override
+    public SQLRuleManager getRuleManager() {
+        return editor.getRuleManager();
+    }
+
     @Override
     public boolean isUseFQNames() {
         return getActivePreferenceStore().getBoolean(SQLPreferenceConstants.PROPOSAL_ALWAYS_FQ);