提交 50a52388 编写于 作者: V Vlad Ilyushchenko

compiler refactoring and join bugfix

上级 86c8a18a
......@@ -166,13 +166,13 @@ public class HashJoinRecordSource extends AbstractCombinedRecordSource implement
sink.putQuoted("op").put(':').putQuoted("HashJoinRecordSource").put(',');
sink.putQuoted("master").put(':').put(master).put(',');
sink.putQuoted("slave").put(':').put(slave).put(',');
sink.putQuoted("joinOn").put(':');
sink.putQuoted("joinOn").put(':').put('[');
sink.put('[');
for (int i = 0, n = masterColumns.size(); i < n; i++) {
if (i > 0) {
sink.put(',');
}
sink.put(masterColumns.getQuick(i).getName());
sink.putQuoted(masterColumns.getQuick(i).getName());
}
sink.put(']').put(',');
sink.put('[');
......@@ -180,10 +180,9 @@ public class HashJoinRecordSource extends AbstractCombinedRecordSource implement
if (i > 0) {
sink.put(',');
}
sink.put(slaveColumns.getQuick(i).getName());
sink.putQuoted(slaveColumns.getQuick(i).getName());
}
sink.put(']');
sink.put('}');
sink.put("]]}");
}
private void buildHashTable() {
......
......@@ -359,16 +359,6 @@ public class QueryCompiler {
}
}
private void analyseLimit(QueryModel model) throws ParserException {
ExprNode lo = model.getLimitLo();
ExprNode hi = model.getLimitHi();
if (hi == null) {
model.setLimitVc(LONG_ZERO_CONST, limitToVirtualColumn(model, lo));
} else {
model.setLimitVc(limitToVirtualColumn(model, lo), limitToVirtualColumn(model, hi));
}
}
private void analyseRegex(QueryModel parent, ExprNode node) throws ParserException {
literalCollectorAIndexes.clear();
literalCollectorBIndexes.clear();
......@@ -393,6 +383,19 @@ public class QueryCompiler {
}
}
private void applyLimit(QueryModel model) throws ParserException {
// analyse limit first as it is easy win
if (model.getLimitLo() != null || model.getLimitHi() != null) {
ExprNode lo = model.getLimitLo();
ExprNode hi = model.getLimitHi();
if (hi == null) {
model.setLimitVc(LONG_ZERO_CONST, limitToVirtualColumn(model, lo));
} else {
model.setLimitVc(limitToVirtualColumn(model, lo), limitToVirtualColumn(model, hi));
}
}
}
@SuppressFBWarnings("PRMC_POSSIBLY_REDUNDANT_METHOD_CALLS")
private void assignFilters(QueryModel parent) throws ParserException {
......@@ -451,7 +454,8 @@ public class QueryCompiler {
}
if (qualifies) {
postFilterRemoved.add(k);
parent.getJoinModels().getQuick(index).setPostJoinWhereClause(filterNodes.getQuick(k));
QueryModel m = parent.getJoinModels().getQuick(index);
m.setPostJoinWhereClause(concatFilters(m.getPostJoinWhereClause(), filterNodes.getQuick(k)));
}
}
}
......@@ -591,16 +595,18 @@ public class QueryCompiler {
RecordSource rs;
if (model.getJoinModels().size() > 1) {
optimiseJoins(model, factory);
rs = compileJoins(model, factory);
} else if (model.getJournalName() != null) {
rs = compileJournal(model, factory);
} else {
rs = compileSingleOrSubQuery(model, factory);
rs = compileSubQuery(model, factory);
}
return limit(order(selectColumns(rs, model), model), model);
}
private RecordSource compileJoins(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
optimiseJoins(model, factory);
ObjList<QueryModel> joinModels = model.getJoinModels();
IntList ordered = model.getOrderedJoinModels();
RecordSource master = null;
......@@ -615,7 +621,8 @@ public class QueryCompiler {
RecordSource slave = m.getRecordSource();
if (slave == null) {
slave = compileSingleOrSubQuery(m, factory);
// subquery would have been compiled already
slave = compileJournal(m, factory);
if (m.getAlias() != null) {
slave.getMetadata().setAlias(m.getAlias().token);
}
......@@ -659,7 +666,9 @@ public class QueryCompiler {
@SuppressWarnings("ConstantConditions")
@SuppressFBWarnings({"CC_CYCLOMATIC_COMPLEXITY"})
private RecordSource compileSingleJournal(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
private RecordSource compileJournal(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
applyLimit(model);
RecordMetadata metadata = model.getMetadata();
JournalMetadata journalMetadata;
......@@ -850,26 +859,15 @@ public class QueryCompiler {
return new JournalSource(ps, rs == null ? new AllRowSource() : rs);
}
private RecordSource compileSingleOrSubQuery(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
// analyse limit first as it is easy win
if (model.getLimitLo() != null || model.getLimitHi() != null) {
analyseLimit(model);
}
if (model.getJournalName() != null) {
return compileSingleJournal(model, factory);
} else {
optimiseSubQueries(model, factory);
return compileSubQuery(model, factory);
}
}
private RecordSource compileSourceInternal(JournalReaderFactory factory, CharSequence query) throws ParserException, JournalException {
return compile(parser.parseInternal(query).getQueryModel(), factory);
}
private RecordSource compileSubQuery(QueryModel model, JournalReaderFactory factory) throws JournalException, ParserException {
applyLimit(model);
optimiseSubQueries(model, factory);
RecordSource rs = compile(model.getNestedModel(), factory);
if (model.getWhereClause() == null) {
return rs;
......@@ -1054,6 +1052,37 @@ public class QueryCompiler {
}
}
private boolean hasNonAggregates(ExprNode node) {
this.exprNodeStack.clear();
// pre-order iterative tree traversal
// see: http://en.wikipedia.org/wiki/Tree_traversal
while (!this.exprNodeStack.isEmpty() || node != null) {
if (node != null) {
switch (node.type) {
case LITERAL:
return true;
case FUNCTION:
if (FunctionFactories.isAggregate(node.token)) {
node = null;
continue;
}
break;
default:
this.exprNodeStack.push(node.rhs);
break;
}
node = node.lhs;
} else {
node = this.exprNodeStack.poll();
}
}
return false;
}
private void homogenizeCrossJoins(QueryModel parent) {
ObjList<QueryModel> joinModels = parent.getJoinModels();
for (int i = 0, n = joinModels.size(); i < n; i++) {
......@@ -1341,10 +1370,10 @@ public class QueryCompiler {
private RecordSource order(RecordSource rs, QueryModel model) throws ParserException {
ObjList<ExprNode> orderBy = model.getOrderBy();
IntList orderByDirection = model.getOrderByDirection();
IntList indices = model.getOrderColumnIndices();
int n = orderBy.size();
if (n > 0) {
IntList orderByDirection = model.getOrderByDirection();
IntList indices = model.getOrderColumnIndices();
RecordMetadata m = rs.getMetadata();
for (int i = 0; i < n; i++) {
ExprNode tok = orderBy.getQuick(i);
......@@ -1779,7 +1808,6 @@ public class QueryCompiler {
RecordSource rs = recordSource;
// if virtual columns are present, create record source to calculate them
if (virtualColumns != null) {
rs = new VirtualColumnRecordSource(rs, virtualColumns);
......
/*******************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
* <p>
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
*
* Copyright (C) 2014-2016 Appsicle
* <p>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
* <p>
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* <p>
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* <p>
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
......@@ -30,6 +30,7 @@
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*
******************************************************************************/
package com.questdb.ql;
......@@ -257,6 +258,16 @@ public class AsOfPartitionedJoinRecordSourceTest extends AbstractOptimiserTest {
}
}
@Test
public void testAnonymousSubqueriesFunc() throws Exception {
try {
compile("select sum(timestamp) from (y) asof join (x) on x.ccy = y.ccy");
} catch (ParserException e) {
Assert.assertEquals(48, QueryError.getPosition());
TestUtils.assertEquals("Invalid journal name/alias", QueryError.getMessage());
}
}
@Test
public void testFixJoin() throws Exception {
final String expected = "2015-03-10T00:01:00.000Z\tSWHYRX\t0.937527447939\tIYMQGYIYHVZMXGRFXUIUNMOQUIHPNGNOTXDHUZFW\t2015-03-10T00:00:50.000Z\tSWHYRX\t0.000039573626\t0.000003805120\tVTJWCP\t-5106801657083469087\t0.2093\t-20638\ttrue\n" +
......
......@@ -77,7 +77,7 @@ public class SubqueryOptimiserTest extends AbstractOptimiserTest {
@Test
public void testJoinSubQueryFilter() throws Exception {
sink.put(compiler.compileSource(factory, "(tab a join tex b on a.id = b.id) a where a.x = 10"));
TestUtils.assertEquals("{\"op\":\"HashJoinRecordSource\",\"master\":{\"op\":\"JournalSource\",\"psrc\":{\"op\":\"JournalPartitionSource\",\"journal\":\"tab\"},\"rsrc\":{\"op\":\"FilteredRowSource\",\"rsrc\":{\"op\":\"AllRowSource\"}}},\"slave\":{\"op\":\"JournalSource\",\"psrc\":{\"op\":\"JournalPartitionSource\",\"journal\":\"tex\"},\"rsrc\":{\"op\":\"AllRowSource\"}},\"joinOn\":[id],[id]}",
TestUtils.assertEquals("{\"op\":\"HashJoinRecordSource\",\"master\":{\"op\":\"JournalSource\",\"psrc\":{\"op\":\"JournalPartitionSource\",\"journal\":\"tab\"},\"rsrc\":{\"op\":\"FilteredRowSource\",\"rsrc\":{\"op\":\"AllRowSource\"}}},\"slave\":{\"op\":\"JournalSource\",\"psrc\":{\"op\":\"JournalPartitionSource\",\"journal\":\"tex\"},\"rsrc\":{\"op\":\"AllRowSource\"}},\"joinOn\":[[\"id\"],[\"id\"]]}",
sink);
}
......
/*******************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
* <p>
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
*
* Copyright (C) 2014-2016 Appsicle
* <p>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
* <p>
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* <p>
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* <p>
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
......@@ -30,10 +30,14 @@
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*
******************************************************************************/
package com.questdb.ql.parser;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonParser;
import com.questdb.ex.JournalException;
import com.questdb.ex.ParserException;
import com.questdb.io.RecordSourcePrinter;
......@@ -57,11 +61,20 @@ public abstract class AbstractOptimiserTest {
protected static final StringSink sink = new StringSink();
protected static final RecordSourcePrinter printer = new RecordSourcePrinter(sink);
private static final AssociativeCache<RecordSource> cache = new AssociativeCache<>(8, 16);
private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
private static final JsonParser jp = new JsonParser();
protected void assertPlan(String expected, String query) throws ParserException, JournalException {
TestUtils.assertEquals(expected, compiler.plan(factory, query));
}
protected void assertPlan2(CharSequence expected, CharSequence query) throws JournalException, ParserException {
sink.clear();
compiler.compileSource(factory, query).toSink(sink);
String s = gson.toJson(jp.parse(sink.toString()));
TestUtils.assertEquals(expected, s);
}
protected void assertThat(String expected, String query, boolean header) throws ParserException, JournalException, IOException {
sink.clear();
RecordSource rs = cache.peek(query);
......
/*******************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
* <p>
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
*
* Copyright (C) 2014-2016 Appsicle
* <p>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
* <p>
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* <p>
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* <p>
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
......@@ -30,6 +30,7 @@
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*
******************************************************************************/
package com.questdb.ql.parser;
......@@ -78,7 +79,7 @@ public class JoinQueryTest extends AbstractOptimiserTest {
@Test
public void testAmbiguousColumn() throws Exception {
try {
assertPlan("", "orders join customers on customerId = customerId");
assertPlan2("", "orders join customers on customerId = customerId");
Assert.fail("Exception expected");
} catch (ParserException e) {
Assert.assertEquals(25, QueryError.getPosition());
......@@ -88,14 +89,122 @@ public class JoinQueryTest extends AbstractOptimiserTest {
@Test
public void testAsOfJoinOrder() throws Exception {
assertPlan("+ 0[ cross ] c\n" +
"+ 1[ asof ] e ON e.employeeId = c.customerId\n" +
"+ 2[ inner ] o ON o.customerId = c.customerId\n" +
"\n",
assertPlan2("{\n" +
" \"op\": \"HashJoinRecordSource\",\n" +
" \"master\": {\n" +
" \"op\": \"AsOfPartitionedJoinRecordSource\",\n" +
" \"master\": {\n" +
" \"op\": \"JournalSource\",\n" +
" \"psrc\": {\n" +
" \"op\": \"JournalPartitionSource\",\n" +
" \"journal\": \"customers\"\n" +
" },\n" +
" \"rsrc\": {\n" +
" \"op\": \"AllRowSource\"\n" +
" }\n" +
" },\n" +
" \"slave\": {\n" +
" \"op\": \"JournalSource\",\n" +
" \"psrc\": {\n" +
" \"op\": \"JournalPartitionSource\",\n" +
" \"journal\": \"employees\"\n" +
" },\n" +
" \"rsrc\": {\n" +
" \"op\": \"AllRowSource\"\n" +
" }\n" +
" },\n" +
" \"masterTsIndex\": 7,\n" +
" \"slaveTsIndex\": 4\n" +
" },\n" +
" \"slave\": {\n" +
" \"op\": \"JournalSource\",\n" +
" \"psrc\": {\n" +
" \"op\": \"JournalPartitionSource\",\n" +
" \"journal\": \"orders\"\n" +
" },\n" +
" \"rsrc\": {\n" +
" \"op\": \"AllRowSource\"\n" +
" }\n" +
" },\n" +
" \"joinOn\": [\n" +
" [\n" +
" \"customerId\"\n" +
" ],\n" +
" [\n" +
" \"customerId\"\n" +
" ]\n" +
" ]\n" +
"}",
"customers c" +
" asof join employees e on c.customerId = e.employeeId" +
" join orders o on c.customerId = o.customerId");
}
@Test
public void testAsOfJoinSubQuery() throws Exception {
assertPlan2("{\n" +
" \"op\": \"HashJoinRecordSource\",\n" +
" \"master\": {\n" +
" \"op\": \"FilteredJournalRecordSource\",\n" +
" \"src\": {\n" +
" \"op\": \"AsOfPartitionedJoinRecordSource\",\n" +
" \"master\": {\n" +
" \"op\": \"JournalSource\",\n" +
" \"psrc\": {\n" +
" \"op\": \"JournalPartitionSource\",\n" +
" \"journal\": \"customers\"\n" +
" },\n" +
" \"rsrc\": {\n" +
" \"op\": \"AllRowSource\"\n" +
" }\n" +
" },\n" +
" \"slave\": {\n" +
" \"op\": \"RBTreeSortedRecordSource\",\n" +
" \"byRowId\": true,\n" +
" \"src\": {\n" +
" \"op\": \"SelectedColumnsRecordSource\",\n" +
" \"src\": {\n" +
" \"op\": \"VirtualColumnRecordSource\",\n" +
" \"src\": {\n" +
" \"op\": \"JournalSource\",\n" +
" \"psrc\": {\n" +
" \"op\": \"JournalPartitionSource\",\n" +
" \"journal\": \"employees\"\n" +
" },\n" +
" \"rsrc\": {\n" +
" \"op\": \"AllRowSource\"\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" },\n" +
" \"masterTsIndex\": 7,\n" +
" \"slaveTsIndex\": 3\n" +
" },\n" +
" \"filter\": \"e.lastName \\u003d \\u0027x\\u0027 and e.blah \\u003d \\u0027y\\u0027\"\n" +
" },\n" +
" \"slave\": {\n" +
" \"op\": \"JournalSource\",\n" +
" \"psrc\": {\n" +
" \"op\": \"JournalPartitionSource\",\n" +
" \"journal\": \"orders\"\n" +
" },\n" +
" \"rsrc\": {\n" +
" \"op\": \"AllRowSource\"\n" +
" }\n" +
" },\n" +
" \"joinOn\": [\n" +
" [\n" +
" \"customerId\"\n" +
" ],\n" +
" [\n" +
" \"customerId\"\n" +
" ]\n" +
" ]\n" +
"}",
"customers c" +
" asof join (select '1' blah, lastName, employeeId, timestamp from employees order by lastName) e on c.customerId = e.employeeId" +
" join orders o on c.customerId = o.customerId where e.lastName = 'x' and e.blah = 'y'");
}
@Test
......@@ -271,7 +380,8 @@ public class JoinQueryTest extends AbstractOptimiserTest {
@Test
public void testInnerJoin() throws Exception {
final String expected = "9619\tWTBHZVPVZZ\tT\tnull\tBMUPYPIZEPQKHZNGZGBUWDS\tPNKVDJOF\tFLRBROMNXKU\t2015-07-10T00:00:09.619Z\t1605271283\t9619\t486\t\t2015-07-10T00:00:29.443Z\tYM\n" +
final String expected = "customerId\tcustomerName\tcontactName\taddress\tcity\tpostalCode\tcountry\ttimestamp\torderId\tcustomerId\tproductId\temployeeId\torderDate\tshipper\n" +
"9619\tWTBHZVPVZZ\tT\tnull\tBMUPYPIZEPQKHZNGZGBUWDS\tPNKVDJOF\tFLRBROMNXKU\t2015-07-10T00:00:09.619Z\t1605271283\t9619\t486\t\t2015-07-10T00:00:29.443Z\tYM\n" +
"9619\tWTBHZVPVZZ\tT\tnull\tBMUPYPIZEPQKHZNGZGBUWDS\tPNKVDJOF\tFLRBROMNXKU\t2015-07-10T00:00:09.619Z\t401073894\t9619\t1645\tDND\t2015-07-10T00:00:31.115Z\tFNMURHFGESODNWN\n" +
"9619\tWTBHZVPVZZ\tT\tnull\tBMUPYPIZEPQKHZNGZGBUWDS\tPNKVDJOF\tFLRBROMNXKU\t2015-07-10T00:00:09.619Z\t921021073\t9619\t1860\tSR\t2015-07-10T00:00:41.263Z\tOJXJCNBLYTOIYI\n" +
"9619\tWTBHZVPVZZ\tT\tnull\tBMUPYPIZEPQKHZNGZGBUWDS\tPNKVDJOF\tFLRBROMNXKU\t2015-07-10T00:00:09.619Z\t1986641415\t9619\t935\tUFUC\t2015-07-10T00:00:50.470Z\tFREQGOPJK\n" +
......@@ -279,7 +389,7 @@ public class JoinQueryTest extends AbstractOptimiserTest {
"9619\tWTBHZVPVZZ\tT\tnull\tBMUPYPIZEPQKHZNGZGBUWDS\tPNKVDJOF\tFLRBROMNXKU\t2015-07-10T00:00:09.619Z\t189633559\t9619\t830\tRQMR\t2015-07-10T00:01:20.166Z\tQPL\n" +
"9619\tWTBHZVPVZZ\tT\tnull\tBMUPYPIZEPQKHZNGZGBUWDS\tPNKVDJOF\tFLRBROMNXKU\t2015-07-10T00:00:09.619Z\t960875992\t9619\t960\t\t2015-07-10T00:01:29.735Z\tYJZPHQDJKOM\n";
assertThat(expected, "customers join orders on customers.customerId = orders.customerId where customerName ~ 'WTBHZVPVZZ'");
assertThat(expected, "customers join orders on customers.customerId = orders.customerId where customerName ~ 'WTBHZVPVZZ'", true);
}
@Test
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册