diff --git a/core/src/main/java/io/questdb/cairo/map/FastMap.java b/core/src/main/java/io/questdb/cairo/map/FastMap.java index b5376349471eba118aa6c3ccf593b5ae750cfbcd..8c7da237eb6e9f88a64f248d9fd3e01e72a8de9e 100644 --- a/core/src/main/java/io/questdb/cairo/map/FastMap.java +++ b/core/src/main/java/io/questdb/cairo/map/FastMap.java @@ -28,6 +28,7 @@ import io.questdb.cairo.sql.Record; import io.questdb.cairo.sql.RecordCursor; import io.questdb.std.*; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; public class FastMap implements Map { @@ -62,7 +63,7 @@ public class FastMap implements Map { public FastMap(int pageSize, @Transient @NotNull ColumnTypes keyTypes, - @Transient @NotNull ColumnTypes valueTypes, + @Transient @Nullable ColumnTypes valueTypes, int keyCapacity, double loadFactor ) { diff --git a/core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java b/core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java index 8498913840fd333324299d60b8ef186ff651e30d..6a43dfc0daf5171260d2fc2f22d987b3be025902 100644 --- a/core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java +++ b/core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java @@ -752,9 +752,14 @@ public class SqlCodeGenerator { } else { return generateTableQuery(model, executionContext); } + } + final RecordCursorFactory factory = generateSubQuery(model, executionContext); + final ExpressionNode filter = model.getWhereClause(); + if (filter != null) { + return new FilteredRecordCursorFactory(factory, functionParser.parseFunction(filter, factory.getMetadata(), executionContext)); } - return generateSubQuery(model, executionContext); + return factory; } private RecordCursorFactory generateOrderBy(RecordCursorFactory recordCursorFactory, QueryModel model) throws SqlException { @@ -975,6 +980,8 @@ public class SqlCodeGenerator { return generateSelectVirtual(model, executionContext); case QueryModel.SELECT_MODEL_ANALYTIC: return generateSelectAnalytic(model, executionContext); + case QueryModel.SELECT_MODEL_DISTINCT: + return generateSelectDistinct(model, executionContext); default: if (model.getJoinModels().size() > 1 && processJoins) { return generateJoins(model, executionContext); @@ -1079,6 +1086,22 @@ public class SqlCodeGenerator { } } + private RecordCursorFactory generateSelectDistinct(QueryModel model, SqlExecutionContext executionContext) throws SqlException { + final RecordCursorFactory factory = generateSubQuery(model, executionContext); + try { + return new DistinctRecordCursorFactory( + configuration, + factory, + entityColumnFilter, + asm + ); + + } catch (CairoException e) { + factory.close(); + throw e; + } + } + private RecordCursorFactory generateSelectVirtual(QueryModel model, SqlExecutionContext executionContext) throws SqlException { assert model.getNestedModel() != null; final RecordCursorFactory factory = generateSubQuery(model, executionContext); diff --git a/core/src/main/java/io/questdb/griffin/SqlOptimiser.java b/core/src/main/java/io/questdb/griffin/SqlOptimiser.java index ad4f93cdc846fdac769ce389757d95def1cbb7a4..8a3e560d239f2139976565a6c86c4292df6bf102 100644 --- a/core/src/main/java/io/questdb/griffin/SqlOptimiser.java +++ b/core/src/main/java/io/questdb/griffin/SqlOptimiser.java @@ -244,7 +244,16 @@ class SqlOptimiser { deletedContexts.add(idx); } - private void addFunction(QueryColumn qc, QueryModel baseModel, QueryModel groupByModel, QueryModel outerModel, QueryModel innerModel, QueryModel analyticModel, QueryModel translatingModel) throws SqlException { + private void addFunction( + QueryColumn qc, + QueryModel baseModel, + QueryModel translatingModel, + QueryModel innerModel, + QueryModel analyticModel, + QueryModel groupByModel, + QueryModel outerModel, + QueryModel distinctModel + ) throws SqlException { // there were no aggregation functions emitted therefore // this is just a function that goes into virtual model innerModel.addColumn(qc); @@ -261,6 +270,7 @@ class SqlOptimiser { groupByModel.addColumn(innerColumn); analyticModel.addColumn(innerColumn); outerModel.addColumn(innerColumn); + distinctModel.addColumn(innerColumn); } private void addJoinContext(QueryModel parent, JoinContext context) { @@ -629,9 +639,10 @@ class SqlOptimiser { QueryModel validatingModel, QueryModel translatingModel, QueryModel innerModel, - QueryModel groupByModel, QueryModel analyticModel, - QueryModel outerModel + QueryModel groupByModel, + QueryModel outerModel, + QueryModel distinctModel ) throws SqlException { // add duplicate column names only to group-by model // taking into account that column is pre-aliased, e.g. @@ -644,10 +655,11 @@ class SqlOptimiser { final CharSequence translatedColumnName = translatingAliasMap.valueAt(index); final CharSequence alias = createColumnAlias(columnName, groupByModel); final QueryColumn translatedColumn = nextColumn(alias, translatedColumnName); + innerModel.addColumn(translatedColumn); groupByModel.addColumn(translatedColumn); analyticModel.addColumn(translatedColumn); outerModel.addColumn(translatedColumn); - innerModel.addColumn(translatedColumn); + distinctModel.addColumn(translatedColumn); } else { final CharSequence alias = createColumnAlias(columnName, translatingModel); addColumnToTranslatingModel( @@ -663,21 +675,23 @@ class SqlOptimiser { // create column that references inner alias we just created innerModel.addColumn(translatedColumn); - groupByModel.addColumn(translatedColumn); analyticModel.addColumn(translatedColumn); + groupByModel.addColumn(translatedColumn); outerModel.addColumn(translatedColumn); + distinctModel.addColumn(translatedColumn); } } private void createSelectColumnsForWildcard( QueryColumn qc, + boolean hasJoins, QueryModel baseModel, QueryModel translatingModel, QueryModel innerModel, - QueryModel groupByModel, QueryModel analyticModel, + QueryModel groupByModel, QueryModel outerModel, - boolean hasJoins + QueryModel distinctModel ) throws SqlException { // this could be a wildcard, such as '*' or 'a.*' int dot = Chars.indexOf(qc.getAst().token, '.'); @@ -690,26 +704,28 @@ class SqlOptimiser { // we are targeting single table createSelectColumnsForWildcard0( baseModel.getJoinModels().getQuick(index), + hasJoins, qc.getAst().position, translatingModel, innerModel, - groupByModel, analyticModel, + groupByModel, outerModel, - hasJoins + distinctModel ); } else { ObjList models = baseModel.getJoinModels(); for (int j = 0, z = models.size(); j < z; j++) { createSelectColumnsForWildcard0( models.getQuick(j), + hasJoins, qc.getAst().position, translatingModel, innerModel, - groupByModel, analyticModel, + groupByModel, outerModel, - hasJoins + distinctModel ); } } @@ -717,13 +733,14 @@ class SqlOptimiser { private void createSelectColumnsForWildcard0( QueryModel srcModel, + boolean hasJoins, int wildcardPosition, QueryModel translatingModel, QueryModel innerModel, - QueryModel groupByModel, QueryModel analyticModel, + QueryModel groupByModel, QueryModel outerModel, - boolean hasJoins + QueryModel distinctModel ) throws SqlException { final ObjList columnNames = srcModel.getColumnNames(); for (int j = 0, z = columnNames.size(); j < z; j++) { @@ -744,9 +761,10 @@ class SqlOptimiser { null, // do not validate translatingModel, innerModel, - groupByModel, analyticModel, - outerModel + groupByModel, + outerModel, + distinctModel ); } } @@ -1866,6 +1884,22 @@ class SqlOptimiser { } } } + + QueryModel nested = base.getNestedModel(); + if (nested != null) { + rewriteOrderByPosition(nested); + } + + ObjList joinModels = base.getJoinModels(); + for (int i = 1, n = joinModels.size(); i < n; i++) { + // we can ignore result of order by rewrite for because + // 1. when join model is not a sub-query it will always have all the fields, so order by wouldn't + // introduce synthetic model (no column needs to be hidden) + // 2. when join model is a sub-query it will have nested model, which can be rewritten. Parent model + // would remain the same again. + rewriteOrderByPosition(joinModels.getQuick(i)); + } + return model; } @@ -1880,7 +1914,7 @@ class SqlOptimiser { * * @param model inbound model * @return outbound model - * @throws SqlException when column names are ambiguos or not found at all. + * @throws SqlException when column names are ambiguous or not found at all. */ private QueryModel rewriteOrderBy(QueryModel model) throws SqlException { // find base model and check if there is "group-by" model in between @@ -1900,7 +1934,7 @@ class SqlOptimiser { final int modelColumnCount = model.getColumns().size(); boolean groupBy = false; - while (base.getColumns().size() > 0) { + while (base.getColumns().size() > 0 && !base.isNestedModelIsSubQuery()) { baseParent = base; base = base.getNestedModel(); groupBy = groupBy || baseParent.getSelectModelType() == QueryModel.SELECT_MODEL_GROUP_BY; @@ -2004,7 +2038,7 @@ class SqlOptimiser { } } } - if (ascendColumns && base != model) { + if (ascendColumns && base != baseParent) { model.addOrderBy(orderBy, base.getOrderByDirection().getQuick(i)); } } @@ -2070,6 +2104,8 @@ class SqlOptimiser { QueryModel groupByModel = queryModelPool.next(); groupByModel.setSelectModelType(QueryModel.SELECT_MODEL_GROUP_BY); + QueryModel distinctModel = queryModelPool.next(); + distinctModel.setSelectModelType(QueryModel.SELECT_MODEL_DISTINCT); QueryModel outerModel = queryModelPool.next(); outerModel.setSelectModelType(QueryModel.SELECT_MODEL_VIRTUAL); QueryModel innerModel = queryModelPool.next(); @@ -2082,10 +2118,10 @@ class SqlOptimiser { boolean useAnalyticModel = false; boolean useGroupByModel = false; boolean useOuterModel = false; + boolean useDistinctModel = model.isDistinct(); final ObjList columns = model.getColumns(); final QueryModel baseModel = model.getNestedModel(); -// baseModel.moveLimitFrom(model); final boolean hasJoins = baseModel.getJoinModels().size() > 1; // sample by clause should be promoted to all of the models as well as validated @@ -2112,13 +2148,32 @@ class SqlOptimiser { if (qc.getAst().type == ExpressionNode.LITERAL) { if (!isNotBindVariable(qc.getAst().token)) { - addFunction(qc, baseModel, groupByModel, outerModel, innerModel, analyticModel, translatingModel); + addFunction( + qc, + baseModel, + translatingModel, + innerModel, + analyticModel, + groupByModel, + outerModel, + distinctModel + ); useInnerModel = true; } else if (Chars.endsWith(qc.getAst().token, '*')) { // in general sense we need to create new column in case // there is change of alias, for example we may have something as simple as // select a.f, b.f from .... - createSelectColumnsForWildcard(qc, baseModel, translatingModel, innerModel, groupByModel, analyticModel, outerModel, hasJoins); + createSelectColumnsForWildcard( + qc, + hasJoins, + baseModel, + translatingModel, + innerModel, + analyticModel, + groupByModel, + outerModel, + distinctModel + ); } else { createSelectColumn( qc.getAlias(), @@ -2126,9 +2181,10 @@ class SqlOptimiser { baseModel, translatingModel, innerModel, - groupByModel, analyticModel, - outerModel + groupByModel, + outerModel, + distinctModel ); } } else { @@ -2152,7 +2208,9 @@ class SqlOptimiser { // group-by column references might be needed when we have // outer model supporting arithmetic such as: // select sum(a)+sum(b) .... - outerModel.addColumn(nextColumn(qc.getAlias())); + QueryColumn aggregateRef = nextColumn(qc.getAlias()); + outerModel.addColumn(aggregateRef); + distinctModel.addColumn(aggregateRef); // pull out literals emitLiterals(qc.getAst(), translatingModel, innerModel, baseModel); useGroupByModel = true; @@ -2167,6 +2225,7 @@ class SqlOptimiser { emitAggregates(qc.getAst(), groupByModel); if (beforeSplit < groupByModel.getColumns().size()) { outerModel.addColumn(qc); + distinctModel.addColumn(nextColumn(qc.getAlias())); // pull literals from newly created group-by columns into both of underlying models for (int j = beforeSplit, n = groupByModel.getColumns().size(); j < n; j++) { @@ -2176,7 +2235,16 @@ class SqlOptimiser { useGroupByModel = true; useOuterModel = true; } else { - addFunction(qc, baseModel, groupByModel, outerModel, innerModel, analyticModel, translatingModel); + addFunction( + qc, + baseModel, + translatingModel, + innerModel, + analyticModel, + groupByModel, + outerModel, + distinctModel + ); useInnerModel = true; } } @@ -2245,6 +2313,11 @@ class SqlOptimiser { root = outerModel; } + if (useDistinctModel) { + distinctModel.setNestedModel(root); + root = distinctModel; + } + if (!useGroupByModel && groupByModel.getSampleBy() != null) { throw SqlException.$(groupByModel.getSampleBy().position, "at least one aggregation function must be present in 'select' clause"); } diff --git a/core/src/main/java/io/questdb/griffin/SqlParser.java b/core/src/main/java/io/questdb/griffin/SqlParser.java index a17dd8a8400a86ddd0e0c49c3c025c6dd145b95d..6e282857d03580d334bf12077d8b676cd0022598 100644 --- a/core/src/main/java/io/questdb/griffin/SqlParser.java +++ b/core/src/main/java/io/questdb/griffin/SqlParser.java @@ -559,12 +559,6 @@ public final class SqlParser { parseSelectClause(lexer, model); } else { lexer.unparse(); - // do not default to wildcard column selection when - // dealing with sub-queries - if (subQueryMode) { - parseFromClause(lexer, model, model); - return model; - } model.addColumn(SqlUtil.nextColumn(queryColumnPool, sqlNodePool, "*", "*")); } QueryModel nestedModel = queryModelPool.next(); @@ -573,22 +567,8 @@ public final class SqlParser { model.setLimit(nestedModel.getLimitLo(), nestedModel.getLimitHi()); nestedModel.setLimit(null, null); } - if ( - nestedModel.getColumns().size() == 0 - && nestedModel.getNestedModel() != null - && nestedModel.getWhereClause() == null - && nestedModel.getSampleBy() == null - && nestedModel.getJoinModels().size() == 1 - && nestedModel.getTimestamp() == null - && nestedModel.getOrderBy().size() == 0 - && nestedModel.getAlias() == null - ) { - nestedModel = nestedModel.getNestedModel(); - } else { - model.setSelectModelType(QueryModel.SELECT_MODEL_CHOOSE); - } + model.setSelectModelType(QueryModel.SELECT_MODEL_CHOOSE); model.setNestedModel(nestedModel); - return model; } @@ -598,6 +578,7 @@ public final class SqlParser { if (Chars.equals(tok, '(')) { model.setNestedModel(parseSubQuery(lexer)); + model.setNestedModelIsSubQuery(true); tok = optTok(lexer); @@ -886,7 +867,12 @@ public final class SqlParser { private void parseSelectClause(GenericLexer lexer, QueryModel model) throws SqlException { while (true) { - CharSequence tok = tok(lexer, "column"); + CharSequence tok = tok(lexer, "[distinct] column"); + + if (Chars.equalsLowerCaseAscii(tok, "distinct")) { + model.setDistinct(true); + tok = tok(lexer, "column"); + } final ExpressionNode expr; // this is quite dramatic workaround for lexer diff --git a/core/src/main/java/io/questdb/griffin/engine/groupby/AbstractSampleByRecordCursorFactory.java b/core/src/main/java/io/questdb/griffin/engine/groupby/AbstractSampleByRecordCursorFactory.java index a608c73f56625ce4620efde735f7df47d75d6302..c00bc672754f42aa79c414dc7c4c4f3fb9ec2091 100644 --- a/core/src/main/java/io/questdb/griffin/engine/groupby/AbstractSampleByRecordCursorFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/groupby/AbstractSampleByRecordCursorFactory.java @@ -87,7 +87,7 @@ public class AbstractSampleByRecordCursorFactory implements RecordCursorFactory recordFunctions, groupByMetadata, keyTypes, - valueTypes, + valueTypes.getColumnCount(), symbolTableIndex, false ); diff --git a/core/src/main/java/io/questdb/griffin/engine/groupby/DistinctRecordCursorFactory.java b/core/src/main/java/io/questdb/griffin/engine/groupby/DistinctRecordCursorFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..297d76afd233cbc7f47bc9d41065b8f8d9fe040c --- /dev/null +++ b/core/src/main/java/io/questdb/griffin/engine/groupby/DistinctRecordCursorFactory.java @@ -0,0 +1,155 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (C) 2014-2019 Appsicle + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + ******************************************************************************/ + +package io.questdb.griffin.engine.groupby; + +import io.questdb.cairo.*; +import io.questdb.cairo.map.Map; +import io.questdb.cairo.map.MapFactory; +import io.questdb.cairo.map.MapKey; +import io.questdb.cairo.sql.*; +import io.questdb.griffin.SqlExecutionContext; +import io.questdb.std.BytecodeAssembler; +import io.questdb.std.Misc; +import io.questdb.std.Transient; +import org.jetbrains.annotations.NotNull; + +public class DistinctRecordCursorFactory implements RecordCursorFactory { + + protected final RecordCursorFactory base; + private final Map dataMap; + private final DistinctRecordCursor cursor; + private final RecordSink mapSink; + // this sink is used to copy recordKeyMap keys to dataMap + private final RecordMetadata metadata; + + public DistinctRecordCursorFactory( + CairoConfiguration configuration, + RecordCursorFactory base, + @Transient @NotNull EntityColumnFilter columnFilter, + @Transient @NotNull BytecodeAssembler asm + ) { + final RecordMetadata metadata = base.getMetadata(); + // sink will be storing record columns to map key + columnFilter.of(metadata.getColumnCount()); + this.mapSink = RecordSinkFactory.getInstance(asm, metadata, columnFilter, false); + this.dataMap = MapFactory.createMap(configuration, metadata); + this.base = base; + this.metadata = metadata; + this.cursor = new DistinctRecordCursor(); + } + + @Override + public void close() { + dataMap.close(); + base.close(); + } + + @Override + public RecordCursor getCursor(SqlExecutionContext executionContext) { + dataMap.clear(); + final RecordCursor baseCursor = base.getCursor(executionContext); + try { + cursor.of(baseCursor, dataMap, mapSink); + return cursor; + } catch (CairoException e) { + baseCursor.close(); + throw e; + } + } + + @Override + public RecordMetadata getMetadata() { + return metadata; + } + + @Override + public boolean isRandomAccessCursor() { + return base.isRandomAccessCursor(); + } + + private static class DistinctRecordCursor implements RecordCursor { + private RecordCursor baseCursor; + private Map dataMap; + private RecordSink recordSink; + private Record record; + + public DistinctRecordCursor() { + } + + @Override + public void close() { + Misc.free(baseCursor); + } + + @Override + public Record getRecord() { + return record; + } + + @Override + public SymbolTable getSymbolTable(int columnIndex) { + return baseCursor.getSymbolTable(columnIndex); + } + + @Override + public boolean hasNext() { + while (baseCursor.hasNext()) { + MapKey key = dataMap.withKey(); + recordSink.copy(record, key); + if (key.create()) { + return true; + } + } + return false; + } + + @Override + public Record newRecord() { + return baseCursor.newRecord(); + } + + @Override + public void recordAt(Record record, long atRowId) { + baseCursor.recordAt(record, atRowId); + } + + @Override + public void recordAt(long rowId) { + baseCursor.recordAt(rowId); + } + + @Override + public void toTop() { + baseCursor.toTop(); + dataMap.clear(); + } + + public void of(RecordCursor baseCursor, Map dataMap, RecordSink recordSink) { + this.baseCursor = baseCursor; + this.dataMap = dataMap; + this.recordSink = recordSink; + this.record = baseCursor.getRecord(); + } + } +} diff --git a/core/src/main/java/io/questdb/griffin/engine/groupby/GroupByRecordCursorFactory.java b/core/src/main/java/io/questdb/griffin/engine/groupby/GroupByRecordCursorFactory.java index 498245eee01fae1b8c2e0e712edacd7099786663..5306e480fb829322fb557d0bf038b7773e6c646c 100644 --- a/core/src/main/java/io/questdb/griffin/engine/groupby/GroupByRecordCursorFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/groupby/GroupByRecordCursorFactory.java @@ -83,7 +83,7 @@ public class GroupByRecordCursorFactory implements RecordCursorFactory { recordFunctions, groupByMetadata, keyTypes, - valueTypes, + valueTypes.getColumnCount(), symbolTableIndex, true ); diff --git a/core/src/main/java/io/questdb/griffin/engine/groupby/GroupByUtils.java b/core/src/main/java/io/questdb/griffin/engine/groupby/GroupByUtils.java index 55d54f00f413ec20326bd146f3f65222fc72cab4..116fccb06de2c0e5dbcfb2838fdf7122e8775fa9 100644 --- a/core/src/main/java/io/questdb/griffin/engine/groupby/GroupByUtils.java +++ b/core/src/main/java/io/questdb/griffin/engine/groupby/GroupByUtils.java @@ -48,7 +48,8 @@ class GroupByUtils { FunctionParser functionParser, SqlExecutionContext executionContext, ObjList groupByFunctions, - ArrayColumnTypes valueTypes) throws SqlException { + ArrayColumnTypes valueTypes + ) throws SqlException { final ObjList columns = model.getColumns(); for (int i = 0, n = columns.size(); i < n; i++) { @@ -82,7 +83,7 @@ class GroupByUtils { ObjList recordFunctions, GenericRecordMetadata groupByMetadata, ArrayColumnTypes keyTypes, - ArrayColumnTypes valueTypes, + int keyColumnIndex, IntIntHashMap symbolTableIndex, boolean timestampUnimportant ) { @@ -96,7 +97,6 @@ class GroupByUtils { final ObjList columns = model.getColumns(); // assert timestampIndex != -1; - int keyColumnIndex = valueTypes.getColumnCount(); int valueColumnIndex = 0; // when we have same column several times in a row diff --git a/core/src/main/java/io/questdb/griffin/engine/groupby/SampleByInterpolateRecordCursorFactory.java b/core/src/main/java/io/questdb/griffin/engine/groupby/SampleByInterpolateRecordCursorFactory.java index cac8edd92a5df402b8a8173249108953cca1e76b..3f58814019a104d06370d70289332a401cdbefcf 100644 --- a/core/src/main/java/io/questdb/griffin/engine/groupby/SampleByInterpolateRecordCursorFactory.java +++ b/core/src/main/java/io/questdb/griffin/engine/groupby/SampleByInterpolateRecordCursorFactory.java @@ -97,7 +97,7 @@ public class SampleByInterpolateRecordCursorFactory implements RecordCursorFacto recordFunctions, groupByMetadata, keyTypes, - valueTypes, + valueTypes.getColumnCount(), symbolTableIndex, false ); diff --git a/core/src/main/java/io/questdb/griffin/model/QueryModel.java b/core/src/main/java/io/questdb/griffin/model/QueryModel.java index 1e56d78dfb8432da9ad8fde1137b2e3d8997d5ae..408736722d2fdb7329c3cc5f0c0acbf2e619e682 100644 --- a/core/src/main/java/io/questdb/griffin/model/QueryModel.java +++ b/core/src/main/java/io/questdb/griffin/model/QueryModel.java @@ -45,6 +45,18 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin public static final int SELECT_MODEL_VIRTUAL = 2; public static final int SELECT_MODEL_ANALYTIC = 3; public static final int SELECT_MODEL_GROUP_BY = 4; + public static final int SELECT_MODEL_DISTINCT = 5; + private static final ObjList modelTypeName = new ObjList<>(); + + static { + modelTypeName.extendAndSet(SELECT_MODEL_NONE, "select"); + modelTypeName.extendAndSet(SELECT_MODEL_CHOOSE, "select-choose"); + modelTypeName.extendAndSet(SELECT_MODEL_VIRTUAL, "select-virtual"); + modelTypeName.extendAndSet(SELECT_MODEL_ANALYTIC, "select-analytic"); + modelTypeName.extendAndSet(SELECT_MODEL_GROUP_BY, "select-group-by"); + modelTypeName.extendAndSet(SELECT_MODEL_DISTINCT, "select-distinct"); + } + private final ObjList columns = new ObjList<>(); private final CharSequenceObjHashMap aliasToColumnMap = new CharSequenceObjHashMap<>(); private final CharSequenceObjHashMap columnToAliasMap = new CharSequenceObjHashMap<>(); @@ -89,7 +101,8 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin private ExpressionNode limitLo; private ExpressionNode limitHi; private int selectModelType = SELECT_MODEL_NONE; - + private boolean nestedModelIsSubQuery = false; + private boolean distinct = false; private QueryModel() { joinModels.add(this); @@ -105,6 +118,14 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin } } + public boolean isDistinct() { + return distinct; + } + + public void setDistinct(boolean distinct) { + this.distinct = distinct; + } + public boolean addAliasIndex(ExpressionNode node, int index) { return aliasIndexes.put(node.token, index); } @@ -199,6 +220,8 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin tableVersion = -1; columnNames.clear(); expressionModels.clear(); + distinct = false; + nestedModelIsSubQuery = false; } public void clearOrderBy() { @@ -247,6 +270,14 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin return -1; } + public boolean isNestedModelIsSubQuery() { + return nestedModelIsSubQuery; + } + + public void setNestedModelIsSubQuery(boolean nestedModelIsSubQuery) { + this.nestedModelIsSubQuery = nestedModelIsSubQuery; + } + public CharSequenceObjHashMap getAliasToColumnMap() { return aliasToColumnMap; } @@ -536,16 +567,7 @@ public class QueryModel implements Mutable, ExecutionModel, AliasTranslator, Sin } private String getSelectModelTypeText() { - switch (selectModelType) { - case SELECT_MODEL_CHOOSE: - return "select-choose"; - case SELECT_MODEL_VIRTUAL: - return "select-virtual"; - case SELECT_MODEL_ANALYTIC: - return "select-analytic"; - default: - return "select-group-by"; - } + return modelTypeName.get(selectModelType); } private void toSink0(CharSink sink, boolean joinSlave) { diff --git a/core/src/test/java/io/questdb/FilesTest.java b/core/src/test/java/io/questdb/FilesTest.java index 100e3ee78f4135545a313ba171c0bb984c5f768d..2514faaa63f4ce7304b5532b6905a5e57b3160e3 100644 --- a/core/src/test/java/io/questdb/FilesTest.java +++ b/core/src/test/java/io/questdb/FilesTest.java @@ -118,9 +118,9 @@ public class FilesTest { @Test public void testListNonExistingDir() { String temp = temporaryFolder.getRoot().getAbsolutePath(); - try (Path path = new Path().of(temp).concat("xyz")) { + try (Path path = new Path().of(temp).concat("xyz").$()) { long pFind = Files.findFirst(path); - Assert.assertEquals(0, pFind); + Assert.assertEquals("failed os=" + Os.errno(), 0, pFind); } } diff --git a/core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java b/core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java index 1186811b347634e9ec34283f3cd558773355e405..1cd69c0dadaf0d771bb28e27099c864406acfbc6 100644 --- a/core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java +++ b/core/src/test/java/io/questdb/griffin/SqlCodeGeneratorTest.java @@ -2264,6 +2264,114 @@ public class SqlCodeGeneratorTest extends AbstractGriffinTest { expected2); } + @Test + public void testSelectDistinct() throws Exception { + final String expected = "a\n" + + "0\n" + + "8\n" + + "3\n" + + "1\n" + + "9\n" + + "2\n" + + "6\n" + + "4\n" + + "7\n" + + "5\n"; + + assertQuery(expected, + "select distinct a from x", + "create table x as " + + "(" + + "select" + + " abs(rnd_int())%10 a" + + " from" + + " long_sequence(20)" + + ")", + null, + "insert into x select * from (" + + "select" + + " abs(rnd_int())%10 a" + + " from long_sequence(1000000)" + + ") ", + expected, true); + } + + @Test + public void testSelectDistinctSymbol() throws Exception { + final String expected = "a\n" + + "EHNRX\n" + + "\n" + + "BHFOW\n" + + "QULOF\n" + + "RUEDR\n" + + "SZSRY\n" + + "YYQE\n" + + "IBBTGP\n" + + "TJWC\n" + + "ZSXU\n" + + "CCXZ\n" + + "KGHVUV\n" + + "SWHYRX\n" + + "OUOJS\n" + + "PDXYSB\n" + + "OOZZ\n" + + "WFFYUD\n" + + "DZJMY\n" + + "GETJ\n" + + "FBVTMH\n" + + "UICW\n"; + + final String expected2 = "a\n" + + "EHNRX\n" + + "\n" + + "BHFOW\n" + + "QULOF\n" + + "RUEDR\n" + + "SZSRY\n" + + "YYQE\n" + + "IBBTGP\n" + + "TJWC\n" + + "ZSXU\n" + + "CCXZ\n" + + "KGHVUV\n" + + "SWHYRX\n" + + "OUOJS\n" + + "PDXYSB\n" + + "OOZZ\n" + + "WFFYUD\n" + + "DZJMY\n" + + "GETJ\n" + + "FBVTMH\n" + + "UICW\n" + + "SSCL\n" + + "HLDN\n" + + "IIB\n" + + "ROGHY\n" + + "CJFT\n" + + "WNX\n" + + "VZKE\n" + + "NDMRS\n" + + "SVNVD\n" + + "ILQP\n"; + + assertQuery(expected, + "select distinct a from x", + "create table x as " + + "(" + + "select" + + " rnd_symbol(20,4,6,2) a" + + " from" + + " long_sequence(10000)" + + ")", + null, + "insert into x select * from (" + + "select" + + " rnd_symbol(10,3,5,0) a" + + " from long_sequence(1000000)" + + ") ", + expected2, true); + } + @Test public void testOrderByAllSupported() throws Exception { final String expected = "a\tb\tc\td\te\tf\tg\ti\tj\tk\tl\tm\tn\n" + diff --git a/core/src/test/java/io/questdb/griffin/SqlParserTest.java b/core/src/test/java/io/questdb/griffin/SqlParserTest.java index 346c83a25b2fc3b00fab8ec848b8f80373ca97a3..9587a8c77d00a4c7df58571543e36a9a151c7a62 100644 --- a/core/src/test/java/io/questdb/griffin/SqlParserTest.java +++ b/core/src/test/java/io/questdb/griffin/SqlParserTest.java @@ -321,10 +321,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testBlockCommentAtMiddle() throws Exception { assertQuery( - "select-choose" + - " x, a" + - " from (" + - "(x where a > 1 and x > 1) 'b a')", + "select-choose x, a from ((select-choose x, a from (x where a > 1 and x > 1)) 'b a')", "(x where /*this is a random comment */a > 1) 'b a' where x > 1", modelOf("x") .col("x", ColumnType.INT) @@ -334,10 +331,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testBlockCommentNested() throws Exception { assertQuery( - "select-choose" + - " x, a" + - " from (" + - "(x where a > 1 and x > 1) 'b a')", + "select-choose x, a from ((select-choose x, a from (x where a > 1 and x > 1)) 'b a')", "(x where a > 1) /* comment /* ok */ whatever */'b a' where x > 1", modelOf("x") .col("x", ColumnType.INT) @@ -347,10 +341,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testBlockCommentUnclosed() throws Exception { assertQuery( - "select-choose" + - " x, a" + - " from (" + - "(x where a > 1 and x > 1) 'b a')", + "select-choose x, a from ((select-choose x, a from (x where a > 1 and x > 1)) 'b a')", "(x where a > 1) 'b a' where x > 1 /* this block comment", modelOf("x") .col("x", ColumnType.INT) @@ -515,7 +506,100 @@ public class SqlParserTest extends AbstractGriffinTest { ); } - // 0x1fb7bd310d95f2a6d9baaf8a8a430a9a04453a8b + @Test + public void testSelectDistinct() throws SqlException { + assertQuery( + "select-distinct a, b from (select-choose a, b from (tab))", + "select distinct a, b from tab", + modelOf("tab") + .col("a", ColumnType.STRING) + .col("b", ColumnType.LONG) + ); + } + + @Test + public void testSelectDistinctArithmetic() throws SqlException { + assertQuery( + "select-distinct column from (select-virtual a + b column from (tab))", + "select distinct a + b from tab", + modelOf("tab") + .col("a", ColumnType.STRING) + .col("b", ColumnType.LONG) + ); + } + + @Test + public void testSelectDistinctGroupByFunction() throws SqlException { + assertQuery( + "select-distinct a, bb from (select-group-by a, sum(b) bb from (tab))", + "select distinct a, sum(b) bb from tab", + modelOf("tab") + .col("a", ColumnType.STRING) + .col("b", ColumnType.LONG) + ); + } + + @Test + public void testSelectDistinctGroupByFunctionArithmetic() throws SqlException { + assertQuery( + "select-distinct a, bb from (select-virtual a, sum1 + sum bb from (select-group-by a, sum(c) sum, sum(b) sum1 from (tab)))", + "select distinct a, sum(b)+sum(c) bb from tab", + modelOf("tab") + .col("a", ColumnType.STRING) + .col("b", ColumnType.LONG) + .col("c", ColumnType.LONG) + ); + } + + @Test + public void testSelectDistinctGroupByFunctionArithmeticLimit() throws SqlException { + assertQuery( + "select-distinct a, bb from (select-virtual a, sum1 + sum bb from (select-group-by a, sum(c) sum, sum(b) sum1 from (tab)) limit 10)", + "select distinct a, sum(b)+sum(c) bb from tab limit 10", + modelOf("tab") + .col("a", ColumnType.STRING) + .col("b", ColumnType.LONG) + .col("c", ColumnType.LONG) + ); + } + + @Test + public void testFilterOnGroupBy() throws SqlException { + assertQuery( + "select-choose hash, count from ((select-group-by hash, count() count from (transactions.csv)) _xQdbA1 where count > 1)", + "select * from (select hash, count() from 'transactions.csv') where count > 1;", + modelOf("transactions.csv").col("hash", ColumnType.LONG256) + ); + } + + @Test + public void testSelectAfterOrderBy() throws SqlException { + assertQuery( + "select-distinct Schema from (select-choose Schema from ((select-virtual Schema, Name, switch(relkind,'r','table','v','view','m','materialized view','i','index','S','sequence','s','special','f','foreign table','p','table','I','index') Type, pg_catalog.pg_get_userbyid(relowner) Owner from (select-choose n.nspname Schema, c.relname Name, c.relkind relkind, c.relowner relowner from (pg_catalog.pg_class() c join (pg_catalog.pg_namespace() n where nspname <> 'pg_catalog' and nspname <> 'information_schema' and nspname !~ '^pg_toast') n on n.oid = c.relnamespace where relkind in ('r','p','v','m','S','f','') and pg_catalog.pg_table_is_visible(oid))) order by Schema, Name) _xQdbA1))", + "select distinct Schema from \n" + + "(SELECT n.nspname as \"Schema\",\n" + + " c.relname as \"Name\",\n" + + " CASE c.relkind\n" + + " WHEN 'r' THEN 'table'\n" + + " WHEN 'v' THEN 'view'\n" + + " WHEN 'm' THEN 'materialized view'\n" + + " WHEN 'i' THEN 'index'\n" + + " WHEN 'S' THEN 'sequence'\n" + + " WHEN 's' THEN 'special'\n" + + " WHEN 'f' THEN 'foreign table'\n" + + " WHEN 'p' THEN 'table'\n" + + " WHEN 'I' THEN 'index' END as \"Type\",\n" + + " pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n" + + "FROM pg_catalog.pg_class c\n" + + " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n" + + "WHERE c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f', '')\n" + + " AND n.nspname <> 'pg_catalog'\n" + + " AND n.nspname <> 'information_schema'\n" + + " AND n.nspname !~ '^pg_toast'\n" + + " AND pg_catalog.pg_table_is_visible(c.oid)\n" + + "ORDER BY 1, 2);\n" + ); + } @Test public void testGroupByWithLimit() throws SqlException { @@ -531,7 +615,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testGroupByWithSubQueryLimit() throws SqlException { assertQuery( - "select-group-by fromAddress, toAddress, count() count from (transactions.csv limit 10000)", + "select-group-by fromAddress, toAddress, count() count from ((select-choose fromAddress, toAddress from (transactions.csv) limit 10000) _xQdbA1)", "select fromAddress, toAddress, count() from ('transactions.csv' limit 10000)", modelOf("transactions.csv") .col("fromAddress", ColumnType.STRING) @@ -552,44 +636,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testConsistentColumnOrder() throws SqlException { assertQuery( - "select-choose" + - " rnd_int," + - " rnd_int1," + - " rnd_boolean," + - " rnd_str," + - " rnd_double," + - " rnd_float," + - " rnd_short," + - " rnd_short1," + - " rnd_date," + - " rnd_timestamp," + - " rnd_symbol," + - " rnd_long," + - " rnd_long1," + - " ts," + - " rnd_byte," + - " rnd_bin" + - " from " + - "(" + - "select-virtual" + - " rnd_int() rnd_int," + - " rnd_int(0,30,2) rnd_int1," + - " rnd_boolean() rnd_boolean," + - " rnd_str(3,3,2) rnd_str," + - " rnd_double(2) rnd_double," + - " rnd_float(2) rnd_float," + - " rnd_short(10,1024) rnd_short," + - " rnd_short() rnd_short1," + - " rnd_date(to_date('2015','yyyy'),to_date('2016','yyyy'),2) rnd_date," + - " rnd_timestamp(to_timestamp('2015','yyyy'),to_timestamp('2016','yyyy'),2) rnd_timestamp," + - " rnd_symbol(4,4,4,2) rnd_symbol," + - " rnd_long(100,200,2) rnd_long," + - " rnd_long() rnd_long1," + - " timestamp_sequence(to_timestamp(0),1000000000) ts," + - " rnd_byte(2,50) rnd_byte," + - " rnd_bin(10,20,2) rnd_bin" + - " from (long_sequence(20))" + - ")", + "select-choose rnd_int, rnd_int1, rnd_boolean, rnd_str, rnd_double, rnd_float, rnd_short, rnd_short1, rnd_date, rnd_timestamp, rnd_symbol, rnd_long, rnd_long1, ts, rnd_byte, rnd_bin from ((select-virtual rnd_int() rnd_int, rnd_int(0,30,2) rnd_int1, rnd_boolean() rnd_boolean, rnd_str(3,3,2) rnd_str, rnd_double(2) rnd_double, rnd_float(2) rnd_float, rnd_short(10,1024) rnd_short, rnd_short() rnd_short1, rnd_date(to_date('2015','yyyy'),to_date('2016','yyyy'),2) rnd_date, rnd_timestamp(to_timestamp('2015','yyyy'),to_timestamp('2016','yyyy'),2) rnd_timestamp, rnd_symbol(4,4,4,2) rnd_symbol, rnd_long(100,200,2) rnd_long, rnd_long() rnd_long1, timestamp_sequence(to_timestamp(0),1000000000) ts, rnd_byte(2,50) rnd_byte, rnd_bin(10,20,2) rnd_bin from (long_sequence(20))) _xQdbA1)", "select * from (select" + " rnd_int()," + " rnd_int(0, 30, 2)," + @@ -1718,17 +1765,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testCrossJoinWithClause() throws SqlException { assertQuery( - "select-choose" + - " c.customerId customerId," + - " c.name name," + - " c.age age," + - " c1.customerId customerId1," + - " c1.name name1," + - " c1.age age1" + - " from (" + - "(customers where name ~= 'X') c" + - " cross join (customers where name ~= 'X' and age = 30) c1" + - ") limit 10", + "select-choose c.customerId customerId, c.name name, c.age age, c1.customerId customerId1, c1.name name1, c1.age age1 from ((select-choose customerId, name, age from (customers where name ~= 'X')) c cross join (select-choose customerId, name, age from (customers where name ~= 'X' and age = 30)) c1) limit 10", "with" + " cust as (customers where name ~= 'X')" + " cust c cross join cust c1 where c1.age = 30 " + @@ -1872,16 +1909,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testEraseColumnPrefixInJoin() throws Exception { assertQuery( - "select-choose" + - " c.customerId customerId," + - " o.customerId customerId1," + - " o.x x" + - " from " + - "(" + - "customers c" + - " outer join (orders o where x = 10 and customerId = 100) o on customerId = c.customerId" + - " where customerId = 100" + - ")", + "select-choose c.customerId customerId, o.customerId customerId1, o.x x from (customers c outer join (select-choose customerId, x from (orders o where x = 10 and customerId = 100)) o on customerId = c.customerId where customerId = 100)", "customers c" + " outer join (orders o where o.x = 10) o on c.customerId = o.customerId" + " where c.customerId = 100", @@ -2311,7 +2339,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testJoin3() throws Exception { assertQuery( - "select-choose x from (select-choose tab2.x x from (tab join tab2 on tab2.x = tab.x cross join tab3 post-join-where f(tab3.x,tab2.x) = tab.x))", + "select-choose x from ((select-choose tab2.x x from (tab join tab2 on tab2.x = tab.x cross join tab3 post-join-where f(tab3.x,tab2.x) = tab.x)) _xQdbA1)", "select x from (select tab2.x from tab join tab2 on tab.x=tab2.x join tab3 on f(tab3.x,tab2.x) = tab.x)", modelOf("tab").col("x", ColumnType.INT), modelOf("tab2").col("x", ColumnType.INT), @@ -2322,7 +2350,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testJoinColumnResolutionOnSubQuery() throws SqlException { assertQuery( - "select-group-by sum(timestamp) sum from ((y) _xQdbA1 cross join (x) _xQdbA2)", + "select-group-by sum(timestamp) sum from ((select-choose ccy, timestamp from (y)) _xQdbA1 cross join (select-choose ccy from (x)) _xQdbA2)", "select sum(timestamp) from (y) cross join (x)", modelOf("x").col("ccy", ColumnType.SYMBOL), modelOf("y").col("ccy", ColumnType.SYMBOL).col("timestamp", ColumnType.TIMESTAMP) @@ -2332,7 +2360,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testJoinColumnResolutionOnSubQuery2() throws SqlException { assertQuery( - "select-group-by sum(timestamp) sum from ((y) _xQdbA1 join (x) _xQdbA2 on _xQdbA2.ccy = _xQdbA1.ccy and _xQdbA2.sym = _xQdbA1.sym)", + "select-group-by sum(timestamp) sum from ((select-choose ccy, timestamp, sym from (y)) _xQdbA1 join (select-choose ccy, sym from (x)) _xQdbA2 on _xQdbA2.ccy = _xQdbA1.ccy and _xQdbA2.sym = _xQdbA1.sym)", "select sum(timestamp) from (y) join (x) on (ccy, sym)", modelOf("x").col("ccy", ColumnType.SYMBOL).col("sym", ColumnType.INT), modelOf("y").col("ccy", ColumnType.SYMBOL).col("timestamp", ColumnType.TIMESTAMP).col("sym", ColumnType.INT) @@ -2342,7 +2370,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testJoinColumnResolutionOnSubQuery3() throws SqlException { assertQuery( - "select-group-by sum(timestamp) sum from ((y) _xQdbA1 cross join x)", + "select-group-by sum(timestamp) sum from ((select-choose ccy, timestamp from (y)) _xQdbA1 cross join x)", "select sum(timestamp) from (y) cross join x", modelOf("x").col("ccy", ColumnType.SYMBOL), modelOf("y").col("ccy", ColumnType.SYMBOL).col("timestamp", ColumnType.TIMESTAMP) @@ -3009,14 +3037,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testJoinWithClausesDefaultAlias() throws SqlException { assertQuery( - "select-choose" + - " cust.customerId customerId," + - " cust.name name," + - " ord.customerId customerId1" + - " from (" + - "(customers where name ~= 'X') cust" + - " outer join (select-choose customerId from (orders where amount > 100)) ord on ord.customerId = cust.customerId" + - " post-join-where ord.customerId != null) limit 10", + "select-choose cust.customerId customerId, cust.name name, ord.customerId customerId1 from ((select-choose customerId, name from (customers where name ~= 'X')) cust outer join (select-choose customerId from (orders where amount > 100)) ord on ord.customerId = cust.customerId post-join-where ord.customerId != null) limit 10", "with" + " cust as (customers where name ~= 'X')," + " ord as (select customerId from orders where amount > 100)" + @@ -3031,14 +3052,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testJoinWithClausesExplicitAlias() throws SqlException { assertQuery( - "select-choose" + - " c.customerId customerId," + - " c.name name," + - " o.customerId customerId1" + - " from ((customers where name ~= 'X') c" + - " outer join (select-choose customerId from (orders where amount > 100)) o on o.customerId = c.customerId" + - " post-join-where o.customerId != null" + - ") limit 10", + "select-choose c.customerId customerId, c.name name, o.customerId customerId1 from ((select-choose customerId, name from (customers where name ~= 'X')) c outer join (select-choose customerId from (orders where amount > 100)) o on o.customerId = c.customerId post-join-where o.customerId != null) limit 10", "with" + " cust as (customers where name ~= 'X')," + " ord as (select customerId from orders where amount > 100)" + @@ -3127,10 +3141,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testLineCommentAtEnd() throws Exception { assertQuery( - "select-choose" + - " x, a" + - " from (" + - "(x where a > 1 and x > 1) 'b a')", + "select-choose x, a from ((select-choose x, a from (x where a > 1 and x > 1)) 'b a')", "(x where a > 1) 'b a' where x > 1\n--this is comment", modelOf("x") .col("x", ColumnType.INT) @@ -3140,10 +3151,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testLineCommentAtMiddle() throws Exception { assertQuery( - "select-choose" + - " x, a" + - " from (" + - "(x where a > 1 and x > 1) 'b a')", + "select-choose x, a from ((select-choose x, a from (x where a > 1 and x > 1)) 'b a')", "(x where a > 1) \n" + " -- this is a comment \n" + "'b a' where x > 1", @@ -3155,10 +3163,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testLineCommentAtStart() throws Exception { assertQuery( - "select-choose" + - " x, a" + - " from (" + - "(x where a > 1 and x > 1) 'b a')", + "select-choose x, a from ((select-choose x, a from (x where a > 1 and x > 1)) 'b a')", "-- hello, this is a comment\n (x where a > 1) 'b a' where x > 1", modelOf("x") .col("x", ColumnType.INT) @@ -3247,25 +3252,36 @@ public class SqlParserTest extends AbstractGriffinTest { " x.orderId orderId," + " x.productId productId," + " y.orderId orderId1," + - " y.customerId customerId" + - " from " + - "(" + - "(" + - "select-choose orders.orderId orderId, products.productId productId" + - " from " + - "(" + + " y.customerId customerId," + + " y.customerId1 customerId1," + + " y.orderId1 orderId11," + + " y.productId productId1," + + " y.supplier supplier," + + " y.productId1 productId11," + + " y.supplier1 supplier1 from (" + + "(select-choose" + + " orders.orderId orderId," + + " products.productId productId from (" + "orders" + " join (orderDetails d where productId = orderId) d on d.orderId = orders.customerId" + " join customers on customers.customerId = orders.customerId" + " join products on products.productId = d.productId" + " join suppliers on suppliers.supplier = products.supplier" + - " where orderId = customerId" + - ")" + - ") x cross join (orders" + - " join customers on customers.customerId = orders.customerId" + + " where orderId = customerId)) x" + + " cross join (" + + "select-choose" + + " orders.orderId orderId," + + " orders.customerId customerId," + + " customers.customerId customerId1," + + " d.orderId orderId1," + + " d.productId productId," + + " suppliers.supplier supplier," + + " products.productId productId1," + + " products.supplier supplier1 from (" + + "orders join customers on customers.customerId = orders.customerId" + " join (orderDetails d where orderId = productId) d on d.productId = orders.orderId" + " join suppliers on suppliers.supplier = orders.orderId" + - " join products on products.productId = orders.orderId and products.supplier = suppliers.supplier) y)", + " join products on products.productId = orders.orderId and products.supplier = suppliers.supplier)) y)", "with x as (select orders.orderId, products.productId from " + "orders" + " join orderDetails d on d.orderId = orders.orderId and d.orderId = customers.customerId" + @@ -3597,7 +3613,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testOrderByOnJoinSubQuery() throws SqlException { assertQuery( - "select-choose x, y from (select-choose a.x x, b.y y, b.s s from ((select-choose x, z from (tab1 where x = 'Z')) a join (tab2 where s ~= 'K') b on b.z = a.z) order by s)", + "select-choose x, y from (select-choose a.x x, b.y y, b.s s from ((select-choose x, z from (tab1 where x = 'Z')) a join (select-choose x, y, z, s from (tab2 where s ~= 'K')) b on b.z = a.z) order by s)", "select a.x, b.y from (select x,z from tab1 where x = 'Z' order by x) a join (tab2 where s ~= 'K') b on a.z=b.z order by b.s", modelOf("tab1") .col("x", ColumnType.INT) @@ -3613,7 +3629,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testOrderByOnJoinSubQuery2() throws SqlException { assertQuery( - "select-choose a.x x, b.y y from ((select-choose x, z from (select-choose x, z, p from (tab1 where x = 'Z') order by p)) a join (tab2 where s ~= 'K') b on b.z = a.z)", + "select-choose a.x x, b.y y from ((select-choose x, z from (select-choose x, z, p from (tab1 where x = 'Z') order by p)) a join (select-choose x, y, z, s from (tab2 where s ~= 'K')) b on b.z = a.z)", "select a.x, b.y from (select x,z from tab1 where x = 'Z' order by p) a join (tab2 where s ~= 'K') b on a.z=b.z", modelOf("tab1") .col("x", ColumnType.INT) @@ -3708,7 +3724,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testSelectColumnsFromJoinSubQueries() throws SqlException { - assertQuery("select-virtual addr, sum_out - sum_in total from ((select-group-by addr, count() count, sum(value) sum_out from (select-choose fromAddress addr, value from (transactions.csv))) a join (select-group-by toAddress, count() count, sum(value) sum_in from (transactions.csv)) b on b.toAddress = a.addr)", + assertQuery("select-virtual addr, sum_out - sum_in total from ((select-choose a.addr addr, a.count count, a.sum_out sum_out, b.toAddress toAddress, b.count count1, b.sum_in sum_in from ((select-group-by addr, count() count, sum(value) sum_out from (select-choose fromAddress addr, value from (transactions.csv))) a join (select-group-by toAddress, count() count, sum(value) sum_in from (transactions.csv)) b on b.toAddress = a.addr)) _xQdbA1)", "select addr, sum_out - sum_in total from (\n" + "(select fromAddress addr, count(), sum(value) sum_out from 'transactions.csv') a join\n" + "(select toAddress, count(), sum(value) sum_in from 'transactions.csv') b on a.addr = b.toAddress\n" + @@ -3720,6 +3736,32 @@ public class SqlParserTest extends AbstractGriffinTest { ); } + @Test + public void testSelectColumnsFromJoinSubQueries2() throws SqlException { + assertQuery("select-choose addr, count, sum_out, toAddress, count1, sum_in from ((select-choose a.addr addr, a.count count, a.sum_out sum_out, b.toAddress toAddress, b.count count1, b.sum_in sum_in from ((select-group-by addr, count() count, sum(value) sum_out from (select-choose fromAddress addr, value from (transactions.csv))) a join (select-group-by toAddress, count() count, sum(value) sum_in from (transactions.csv)) b on b.toAddress = a.addr)) _xQdbA1)", + "(\n" + + "(select fromAddress addr, count(), sum(value) sum_out from 'transactions.csv') a join\n" + + "(select toAddress, count(), sum(value) sum_in from 'transactions.csv') b on a.addr = b.toAddress\n" + + ")", + modelOf("transactions.csv") + .col("fromAddress", ColumnType.LONG) + .col("toAddress", ColumnType.LONG) + .col("value", ColumnType.LONG) + ); + } + + @Test + public void testSelectColumnsFromJoinSubQueries3() throws SqlException { + assertQuery("select-choose a.addr addr, a.count count, a.sum_out sum_out, b.toAddress toAddress, b.count count1, b.sum_in sum_in from ((select-group-by addr, count() count, sum(value) sum_out from (select-choose fromAddress addr, value from (transactions.csv))) a join (select-group-by toAddress, count() count, sum(value) sum_in from (transactions.csv)) b on b.toAddress = a.addr)", + "(select fromAddress addr, count(), sum(value) sum_out from 'transactions.csv') a join\n" + + "(select toAddress, count(), sum(value) sum_in from 'transactions.csv') b on a.addr = b.toAddress\n", + modelOf("transactions.csv") + .col("fromAddress", ColumnType.LONG) + .col("toAddress", ColumnType.LONG) + .col("value", ColumnType.LONG) + ); + } + @Test public void testOrderByOnOuterResult() throws SqlException { assertQuery( @@ -3745,7 +3787,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testOrderByWithSampleBy() throws SqlException { assertQuery( - "select-group-by a, sum(b) sum from ((tab order by t) _xQdbA1) timestamp (t) sample by 2m order by a", + "select-group-by a, sum(b) sum from ((select-choose a, b, t from (tab) order by t) _xQdbA1) timestamp (t) sample by 2m order by a", "select a, sum(b) from (tab order by t) timestamp(t) sample by 2m order by a", modelOf("tab") .col("a", ColumnType.INT) @@ -3757,7 +3799,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testOrderByWithSampleBy2() throws SqlException { assertQuery( - "select-group-by a, sum(b) sum from ((select-group-by a, sum(b) b from ((tab order by t) _xQdbA3) timestamp (t) sample by 10m) _xQdbA1) timestamp (t) order by a", + "select-group-by a, sum(b) sum from ((select-group-by a, sum(b) b from ((select-choose a, b, t from (tab) order by t) _xQdbA3) timestamp (t) sample by 10m) _xQdbA1) timestamp (t) order by a", "select a, sum(b) from (select a,sum(b) b from (tab order by t) timestamp(t) sample by 10m order by t) order by a", modelOf("tab") .col("a", ColumnType.INT) @@ -3983,7 +4025,7 @@ public class SqlParserTest extends AbstractGriffinTest { assertSyntaxError( "select a, sum(b) from ((tab order by t) timestamp(t) sample by 10m order by t) order by a", 63, - "'sample by' must be used with 'select'", + "at least one aggregation function must be present", modelOf("tab") .col("a", ColumnType.INT) .col("b", ColumnType.INT) @@ -4071,7 +4113,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testSelectFromSubQuery() throws SqlException { assertQuery( - "select-choose x from ((tab where y > 10) a)", + "select-choose x from ((select-choose x, y from (tab where y > 10)) a)", "select a.x from (tab where y > 10) a", modelOf("tab") .col("x", ColumnType.INT) @@ -4272,7 +4314,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testSimpleSubQuery() throws Exception { assertQuery( - "select-choose y from ((x where y > 1) _xQdbA1)", + "select-choose y from ((select-choose y from (x where y > 1)) _xQdbA1)", "(x) where y > 1", modelOf("x").col("y", ColumnType.INT) ); @@ -4495,10 +4537,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testSubQueryAliasWithSpace() throws Exception { assertQuery( - "select-choose" + - " x, a" + - " from (" + - "(x where a > 1 and x > 1) 'b a')", + "select-choose x, a from ((select-choose x, a from (x where a > 1 and x > 1)) 'b a')", "(x where a > 1) 'b a' where x > 1", modelOf("x") .col("x", ColumnType.INT) @@ -4624,7 +4663,7 @@ public class SqlParserTest extends AbstractGriffinTest { @Test public void testTimestampOnSubQuery() throws Exception { - assertQuery("select-choose x from ((a b where x > y) _xQdbA1) timestamp (x)", + assertQuery("select-choose x from ((select-choose x, y from (a b where x > y)) _xQdbA1) timestamp (x)", "select x from (a b) timestamp(x) where x > y", modelOf("a").col("x", ColumnType.INT).col("y", ColumnType.INT)); } @@ -4795,6 +4834,10 @@ public class SqlParserTest extends AbstractGriffinTest { createModelsAndRun(() -> { sink.clear(); ExecutionModel model = compiler.testCompileModel(query, sqlExecutionContext); +// RecordCursorFactory factory = compiler.compile(query); +// if (factory != null) { +// factory.close(); +// } Assert.assertEquals(model.getModelType(), modelType); ((Sinkable) model).toSink(sink); TestUtils.assertEquals(expected, sink);