subquery optimisation for nested subqueries

core/src/main/java/com/questdb/ql/impl/select/SelectedColumnsMetadata.java

@@ -41,6 +41,7 @@ import com.questdb.misc.Unsafe;
 import com.questdb.std.CharSequenceHashSet;
 import com.questdb.std.CharSequenceIntHashMap;
 import com.questdb.std.ObjList;
+import com.questdb.std.Transient;
 import com.questdb.store.ColumnType;
 
 import java.util.Arrays;
@@ -64,7 +65,7 @@ class SelectedColumnsMetadata extends AbstractRecordMetadata {
      * @param names    list of column names to select
      * @param aliases  set of column aliases
      */
-    SelectedColumnsMetadata(RecordMetadata delegate, ObjList<CharSequence> names, CharSequenceHashSet aliases) {
+    SelectedColumnsMetadata(RecordMetadata delegate, @Transient ObjList<CharSequence> names, @Transient CharSequenceHashSet aliases) {
         this.delegate = delegate;
         int k = names.size();
         this.nameIndex = new CharSequenceIntHashMap(k);

core/src/main/java/com/questdb/ql/impl/select/SelectedColumnsRecordSource.java

@@ -46,6 +46,7 @@ import com.questdb.ql.ops.AbstractCombinedRecordSource;
 import com.questdb.std.CharSequenceHashSet;
 import com.questdb.std.CharSink;
 import com.questdb.std.ObjList;
+import com.questdb.std.Transient;
 
 public class SelectedColumnsRecordSource extends AbstractCombinedRecordSource {
     private final RecordSource recordSource;
@@ -54,7 +55,7 @@ public class SelectedColumnsRecordSource extends AbstractCombinedRecordSource {
     private final SelectedColumnsStorageFacade storageFacade;
     private RecordCursor recordCursor;
 
-    public SelectedColumnsRecordSource(RecordSource recordSource, ObjList<CharSequence> names, CharSequenceHashSet aliases) {
+    public SelectedColumnsRecordSource(RecordSource recordSource, @Transient ObjList<CharSequence> names, @Transient CharSequenceHashSet aliases) {
         this.recordSource = recordSource;
         RecordMetadata dm = recordSource.getMetadata();
         this.metadata = new SelectedColumnsMetadata(dm, names, aliases);
@@ -62,7 +63,7 @@ public class SelectedColumnsRecordSource extends AbstractCombinedRecordSource {
         this.storageFacade = new SelectedColumnsStorageFacade(dm, metadata, names);
     }
 
-    public SelectedColumnsRecordSource(RecordSource recordSource, ObjList<CharSequence> names) {
+    public SelectedColumnsRecordSource(RecordSource recordSource, @Transient ObjList<CharSequence> names) {
         this.recordSource = recordSource;
         RecordMetadata dm = recordSource.getMetadata();
         this.metadata = new SelectedColumnsMetadata(dm, names);

core/src/main/java/com/questdb/ql/impl/select/SelectedColumnsStorageFacade.java

 /*******************************************************************************
- * ___                  _   ____  ____
- * / _ \ _   _  ___  ___| |_|  _ \| __ )
- * | | | | | | |/ _ \/ __| __| | | |  _ \
- * | |_| | |_| |  __/\__ \ |_| |_| | |_) |
- * \__\_\\__,_|\___||___/\__|____/|____/
- * <p>
+ *    ___                  _   ____  ____
+ *   / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *  | | | | | | |/ _ \/ __| __| | | |  _ \
+ *  | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *   \__\_\\__,_|\___||___/\__|____/|____/
+ *
  * Copyright (C) 2014-2016 Appsicle
- * <p>
+ *
  * This program is free software: you can redistribute it and/or  modify
  * it under the terms of the GNU Affero General Public License, version 3,
  * as published by the Free Software Foundation.
- * <p>
+ *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU Affero General Public License for more details.
- * <p>
+ *
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- * <p>
+ *
  * As a special exception, the copyright holders give permission to link the
  * code of portions of this program with the OpenSSL library under certain
  * conditions as described in each individual source file and distribute
@@ -30,6 +30,7 @@
  * delete this exception statement from your version. If you delete this
  * exception statement from all source files in the program, then also delete
  * it in the license file.
+ *
  ******************************************************************************/
 
 package com.questdb.ql.impl.select;
@@ -38,6 +39,7 @@ import com.questdb.factory.JournalReaderFactory;
 import com.questdb.factory.configuration.RecordMetadata;
 import com.questdb.ql.StorageFacade;
 import com.questdb.std.ObjList;
+import com.questdb.std.Transient;
 import com.questdb.store.SymbolTable;
 
 public class SelectedColumnsStorageFacade implements StorageFacade {
@@ -46,7 +48,7 @@ public class SelectedColumnsStorageFacade implements StorageFacade {
     private final RecordMetadata metadata;
     private StorageFacade delegate;
 
-    public SelectedColumnsStorageFacade(RecordMetadata parentMetadata, RecordMetadata metadata, ObjList<CharSequence> names) {
+    public SelectedColumnsStorageFacade(RecordMetadata parentMetadata, RecordMetadata metadata, @Transient ObjList<CharSequence> names) {
         this.metadata = metadata;
         int k = names.size();
         this.reindex = new int[k];

core/src/main/java/com/questdb/ql/parser/QueryCompiler.java

@@ -559,17 +559,56 @@ public class QueryCompiler {
         return factory.getOrCreateMetadata(new JournalKey<>(reader));
     }
 
+    private void collectSelectedColumns(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
+        selectedColumnAliases.clear();
+
+        // ok, there are selected columns
+        // literal columns will be narrowing down journal columns
+        // constant columns are interesting from narrowing down constant conditions point of view
+        // e.g. (select 'a' c from B) where c = 'b'
+        // this is the same as:
+        // select 'a' c from B where 'a' = 'c', which is intrinsic FALSE
+        int n = model.getColumns().size();
+        if (n > 0) {
+            for (int i = 0; i < n; i++) {
+                QueryColumn qc = model.getColumns().getQuick(i);
+                switch (qc.getAst().type) {
+                    case CONSTANT:
+                        if (qc.getAlias() != null) {
+                            selectedColumnAliases.add(qc.getAlias());
+                        }
+                        break;
+                    case LITERAL:
+                        if (qc.getAlias() != null) {
+                            selectedColumnAliases.add(qc.getAlias());
+                        } else {
+                            selectedColumnAliases.add(qc.getAst().token);
+                        }
+                        break;
+                    default:
+                        break;
+                }
+            }
+        } else if (model.getJournalName() != null) {
+            JournalMetadata m = collectJournalMetadata(model, factory);
+            for (int i = 0, k = m.getColumnCount(); i < k; i++) {
+                selectedColumnAliases.add(m.getColumnName(i));
+            }
+        } else {
+            collectSelectedColumns(model.getNestedModel(), factory);
+        }
+    }
+
     private RecordSource compile(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
-        return limit(
-                order(
-                        selectColumns(
-                                model.getJoinModels().size() > 1 ?
-                                        optimise(model, factory).compileJoins(model, factory) :
-                                        optimiseSubQueries(model, factory).compileSingleOrSubQuery(model, factory),
-                                model
-                        ), model
-                ), model
-        );
+        RecordSource rs;
+
+        if (model.getJoinModels().size() > 1) {
+            optimiseJoins(model, factory);
+            rs = compileJoins(model, factory);
+        } else {
+            rs = compileSingleOrSubQuery(model, factory);
+        }
+        return limit(order(selectColumns(rs, model), model), model);
     }
 
     private RecordSource compileJoins(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
@@ -829,7 +868,12 @@ public class QueryCompiler {
             analyseLimit(model);
         }
 
-        return model.getJournalName() != null ? compileSingleJournal(model, factory) : compileSubQuery(model, factory);
+        if (model.getJournalName() != null) {
+            return compileSingleJournal(model, factory);
+        } else {
+            optimiseSubQueries(model, factory);
+            return compileSubQuery(model, factory);
+        }
     }
 
     private RecordSource compileSourceInternal(JournalReaderFactory factory, CharSequence query) throws ParserException, JournalException {
@@ -1236,9 +1280,10 @@ public class QueryCompiler {
         return result;
     }
 
-    private QueryCompiler optimise(QueryModel parent, JournalReaderFactory factory) throws JournalException, ParserException {
+    private void optimiseJoins(QueryModel parent, JournalReaderFactory factory) throws JournalException, ParserException {
         ObjList<QueryModel> joinModels = parent.getJoinModels();
 
+        // todo: remove check, this method must not be called on single journal SQLs
         int n = joinModels.size();
         if (n > 1) {
 
@@ -1279,63 +1324,21 @@ public class QueryCompiler {
             alignJoinClauses(parent);
             addTransitiveFilters(parent);
         }
-        return this;
     }
 
     // todo: this is quite primitive (first cut), joins and multi-level subqueries
-    private QueryCompiler optimiseSubQueries(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
+    private void optimiseSubQueries(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
         QueryModel nm = model.getNestedModel();
-
-        if (nm == null || nm.getJournalName() == null) {
-            return this;
-        }
-
-        CharSequenceHashSet names = new CharSequenceHashSet();
-
-        // ok, there are selected columns
-        // literal columns will be narrowing down journal columns
-        // constant columns are interesting from narrowing down constant conditions point of view
-        // e.g. (select 'a' c from B) where c = 'b'
-        // this is the same as:
-        // select 'a' c from B where 'a' = 'c', which is intrinsic FALSE
-        int n = nm.getColumns().size();
-        if (n > 0) {
-            for (int i = 0; i < n; i++) {
-                QueryColumn qc = nm.getColumns().getQuick(i);
-                switch (qc.getAst().type) {
-                    case CONSTANT:
-                        if (qc.getAlias() != null) {
-                            names.add(qc.getAlias());
-                        }
-                        break;
-                    case LITERAL:
-                        if (qc.getAlias() != null) {
-                            names.add(qc.getAlias());
-                        } else {
-                            names.add(qc.getAst().token);
-                        }
-                        break;
-                    default:
-                        break;
-                }
-            }
-        } else {
-            JournalMetadata m = collectJournalMetadata(nm, factory);
-            for (int i = 0, k = m.getColumnCount(); i < k; i++) {
-                names.add(m.getColumnName(i));
-            }
-        }
-
+        collectSelectedColumns(nm, factory);
         processAndConditions(model, model.getWhereClause());
         literalMatcher.of(model.getAlias() != null ? model.getAlias().token : null);
 
         ExprNode nmWhere = nm.getWhereClause();
         ExprNode thisWhere = null;
         ObjList<ExprNode> w = model.getParsedWhere();
-        n = w.size();
-        for (int i = 0; i < n; i++) {
+        for (int i = 0, n = w.size(); i < n; i++) {
             ExprNode node = w.getQuick(i);
-            if (literalMatcher.matches(node, names)) {
+            if (literalMatcher.matches(node, selectedColumnAliases)) {
                 nmWhere = concatFilters(nmWhere, node);
             } else {
                 thisWhere = concatFilters(thisWhere, node);
@@ -1345,7 +1348,9 @@ public class QueryCompiler {
         nm.setWhereClause(nmWhere);
         model.setWhereClause(thisWhere);
 
-        return optimiseSubQueries(nm, factory);
+        if (nm.getNestedModel() != null) {
+            optimiseSubQueries(nm, factory);
+        }
     }
 
     private RecordSource order(RecordSource rs, QueryModel model) throws ParserException {
@@ -1378,6 +1383,7 @@ public class QueryCompiler {
         }
     }
 
+    // todo: remove
     CharSequence plan(JournalReaderFactory factory, CharSequence query) throws ParserException, JournalException {
         QueryModel model = parser.parse(query).getQueryModel();
         resetAndOptimise(model, factory);
@@ -1644,7 +1650,7 @@ public class QueryCompiler {
 
     private void resetAndOptimise(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
         clearState();
-        optimise(model, factory);
+        optimiseJoins(model, factory);
     }
 
     private void resolveJoinMetadata(QueryModel parent, int index, JournalReaderFactory factory) throws JournalException, ParserException {

core/src/test/java/com/questdb/ql/SubqueryOptimiserTest.java

@@ -41,7 +41,6 @@ import com.questdb.ql.parser.AbstractOptimiserTest;
 import com.questdb.test.tools.TestUtils;
 import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Test;
 
 public class SubqueryOptimiserTest extends AbstractOptimiserTest {
@@ -88,7 +87,6 @@ public class SubqueryOptimiserTest extends AbstractOptimiserTest {
     }
 
     @Test
-    @Ignore
     public void testRecursiveAliasedSubquery() throws Exception {
         sink.put(compiler.compileSource(factory, "((tab order by x) a where a.x = 10) b where b.y > 100"));
         TestUtils.assertEquals("{\"op\":\"RBTreeSortedRecordSource\",\"byRowId\":true,\"src\":{\"op\":\"JournalSource\",\"psrc\":{\"op\":\"JournalPartitionSource\",\"journal\":\"tab\"},\"rsrc\":{\"op\":\"FilteredRowSource\",\"rsrc\":{\"op\":\"AllRowSource\"}}}}",

roadmap.md

@@ -19,6 +19,8 @@ __lang__
 - [x] query parameters
 - [x] support for comments (both block /* */ and line --)
 - [ ] analytical clauses
+- [ ] __subquery optimiser (in progress)__
+- [ ] order by optimiser
 
 __server__
 
@@ -36,6 +38,7 @@ __server__
 - [x] C layer for linux (epoll)
 - [x] C layer for mac/bsd (kqueue)
 - [ ] MySQL wire protocol implementation
+- [ ] sql result export to delimited format
 
 __misc__