ignite-3860 - fixes2
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/bf337bc5 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/bf337bc5 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/bf337bc5 Branch: refs/heads/master Commit: bf337bc53bca0a1a0ab6b8b5d4f19e509287ebe9 Parents: 207a669 Author: Sergi Vladykin <[email protected]> Authored: Tue Feb 21 18:53:16 2017 +0300 Committer: Sergi Vladykin <[email protected]> Committed: Tue Feb 21 18:53:16 2017 +0300 ---------------------------------------------------------------------- .../query/h2/sql/GridSqlQuerySplitter.java | 52 ++++++++++++------ .../query/h2/sql/GridSqlStatement.java | 5 ++ .../query/h2/sql/GridSqlSubquery.java | 2 +- .../h2/sql/AbstractH2CompareQueryTest.java | 21 +++----- .../query/h2/sql/H2CompareBigQueryTest.java | 56 +++++++++++++------- .../processors/query/h2/sql/bigQuery.sql | 2 +- 6 files changed, 86 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java index 165b6b8..a215f20 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java @@ -21,6 +21,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; +import java.util.BitSet; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -35,7 +36,9 @@ import org.apache.ignite.IgniteException; import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery; import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery; import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; import org.h2.command.Prepared; import org.h2.command.dml.Query; import org.h2.command.dml.SelectUnion; @@ -407,15 +410,16 @@ public class GridSqlQuerySplitter { * @param qrym Query model. */ private void injectSortingFirstJoin(QueryModel qrym) { - // Must always be generated unique aliases here. - GridSqlAlias leftTbl = qrym.get(0).ast(); - GridSqlAlias rightTbl = qrym.get(1).ast(); - GridSqlJoin join = findJoin(qrym, 0); // We are at the beginning, thus left and right AST must be children of the same join AST. - assert join.leftTable() == leftTbl: join.getSQL(); - assert join.rightTable() == rightTbl: join.getSQL(); + // join2 + // / \ + // join1 \ + // / \ \ + // T0 T1 T2 + GridSqlAlias leftTbl = (GridSqlAlias)join.leftTable(); + GridSqlAlias rightTbl = (GridSqlAlias)join.rightTable(); // Collect all AND conditions. List<AndCondition> andConditions = new ArrayList<>(); @@ -461,10 +465,15 @@ public class GridSqlQuerySplitter { * @param orderByCols Columns for ORDER BY. */ private void injectOrderBy(GridSqlAlias subQryAlias, List<GridSqlColumn> orderByCols) { + if (orderByCols.isEmpty()) + return; + // Structure: alias -> subquery -> query - GridSqlQuery qry = subQryAlias.child().child(); + GridSqlQuery qry = GridSqlAlias.<GridSqlSubquery>unwrap(subQryAlias).subquery(); GridSqlSelect select = leftmostSelect(qry); // The leftmost SELECT in UNION defines column names. + BitSet set = new BitSet(); + for (int i = 0; i < orderByCols.size(); i++) { GridSqlColumn col = orderByCols.get(i); @@ -480,12 +489,21 @@ public class GridSqlQuerySplitter { else if (expr instanceof GridSqlColumn) colName = ((GridSqlColumn)expr).columnName(); else - throw new IllegalStateException(); // It must be impossible to join by this column then. + // It must be impossible to join by this column then, because the expression has no name. + throw new IllegalStateException(); if (colName.equals(col.columnName())) break; // Found the needed column index. + + colIdx++; } + // Avoid duplicates. + if (set.get(colIdx)) + continue; + + set.set(colIdx, true); + // Add sort column to the query. qry.addSort(new GridSqlSortColumn(colIdx, true, false, false)); } @@ -591,8 +609,6 @@ public class GridSqlQuerySplitter { for (int i = begin; i <= end; i++) { QueryModel child = qrym.get(i); - assert !child.needSplit && !child.needSplitChild; - wrapQrym.add(child); } @@ -620,7 +636,7 @@ public class GridSqlQuerySplitter { int end, GridSqlAlias wrapAlias ) { - GridSqlSelect wrapSelect = wrapAlias.child(); + GridSqlSelect wrapSelect = GridSqlAlias.<GridSqlSubquery>unwrap(wrapAlias).subquery(); final int last = qrym.size() - 1; @@ -887,7 +903,7 @@ public class GridSqlQuerySplitter { if (select.where() == null) return; - GridSqlSelect wrapSelect = wrapAlias.child(); + GridSqlSelect wrapSelect = GridSqlAlias.<GridSqlSubquery>unwrap(wrapAlias).subquery(); List<AndCondition> andConditions = new ArrayList<>(); @@ -932,7 +948,7 @@ public class GridSqlQuerySplitter { } else { for (int i = 0; i < ast.size(); i++) { - if (!isAllRelatedToTables(tblAliases, ast)) + if (!isAllRelatedToTables(tblAliases, ast.child(i))) return false; } } @@ -995,8 +1011,6 @@ public class GridSqlQuerySplitter { for (int i = qrym.size() - 1; i > idx; i--) join = (GridSqlJoin)join.leftTable(); - assert join.rightTable() == qrym.get(idx).ast(); - return join; } @@ -1956,6 +1970,7 @@ public class GridSqlQuerySplitter { */ private static final class QueryModel extends ArrayList<QueryModel> { /** */ + @GridToStringInclude final Type type; /** */ @@ -1968,9 +1983,11 @@ public class GridSqlQuerySplitter { int childIdx; /** If it is a SELECT and we need to split it. Makes sense only for type SELECT. */ + @GridToStringInclude boolean needSplit; /** If we have a child SELECT that we should split. */ + @GridToStringInclude boolean needSplitChild; /** If this is UNION ALL. Makes sense only for type UNION.*/ @@ -2003,6 +2020,11 @@ public class GridSqlQuerySplitter { private boolean isQuery() { return type == Type.SELECT || type == Type.UNION; } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(QueryModel.class, this); + } } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java index 21cf596..f3acfb9 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlStatement.java @@ -31,6 +31,11 @@ public abstract class GridSqlStatement { */ public abstract String getSQL(); + /** {@inheritDoc} */ + @Override public String toString() { + return getSQL(); + } + /** * @param explain Explain. * @return {@code this}. http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java index 887e427..f073e7d 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlSubquery.java @@ -41,7 +41,7 @@ public class GridSqlSubquery extends GridSqlElement { /** * @return Subquery AST. */ - public GridSqlQuery subquery() { + public <X extends GridSqlQuery> X subquery() { return child(0); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java index f0ab123..07c8c5e 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/AbstractH2CompareQueryTest.java @@ -25,12 +25,10 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; -import junit.framework.AssertionFailedError; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; @@ -39,7 +37,6 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode; import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.marshaller.optimized.OptimizedMarshaller; import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; @@ -316,15 +313,7 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest setDistributedJoins(distrib). setEnforceJoinOrder(enforceJoinOrder)).getAll(); - try { - assertRsEquals(h2Res, cacheRes, ordering); - } - catch (AssertionFailedError e) { - X.println("Sql query:\n" + sql + "\nargs=" + Arrays.toString(args)); - X.println("[h2Res=" + h2Res + ", cacheRes=" + cacheRes + "]"); - - throw e; - } + assertRsEquals(h2Res, cacheRes, ordering); return h2Res; } @@ -407,7 +396,9 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest Iterator<Map.Entry<String,Integer>> iter1 = rowsWithCnt1.entrySet().iterator(); Iterator<Map.Entry<String,Integer>> iter2 = rowsWithCnt2.entrySet().iterator(); - for (;;) { + int uSize = rowsWithCnt1.size(); + + for (int i = 0;; i++) { if (!iter1.hasNext()) { assertFalse(iter2.hasNext()); @@ -419,8 +410,8 @@ public abstract class AbstractH2CompareQueryTest extends GridCommonAbstractTest Map.Entry<String, Integer> e1 = iter1.next(); Map.Entry<String, Integer> e2 = iter2.next(); - assertEquals(e1.getKey(), e2.getKey()); - assertEquals(e1.getValue(), e2.getValue()); + assertEquals("Key " + i + " of " + uSize, e1.getKey(), e2.getKey()); + assertEquals("Count " + i + " of " + uSize, e1.getValue(), e2.getValue()); } break; http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java index dc73e96..f8526a8 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/sql/H2CompareBigQueryTest.java @@ -31,8 +31,10 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.affinity.AffinityKey; +import org.apache.ignite.cache.query.SqlFieldsQuery; import org.apache.ignite.cache.query.annotations.QuerySqlField; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.util.typedef.X; /** * Executes one big query (and subqueries of the big query) to compare query results from h2 database instance and @@ -54,7 +56,7 @@ import org.apache.ignite.configuration.CacheConfiguration; * - date - | - date - | * - alias - | - alias - | * - archSeq - | - archSeq - ------------------- | - * ------------------ | --------------------- ----repl.Exec------ | + * ------------------ | --------------------- ----part.Exec------ | * | ------------------- | * ----------------- | - rootOrderId PK - ---- * ---part.Cancel--- | - date - @@ -119,19 +121,15 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest { /** {@inheritDoc} */ @Override protected void setIndexedTypes(CacheConfiguration<?, ?> cc, CacheMode mode) { - if (mode == CacheMode.PARTITIONED) + if (mode == CacheMode.PARTITIONED) { cc.setIndexedTypes( Integer.class, CustOrder.class, useColocatedData() ? AffinityKey.class : Integer.class, ReplaceOrder.class, useColocatedData() ? AffinityKey.class : Integer.class, OrderParams.class, - useColocatedData() ? AffinityKey.class : Integer.class, Cancel.class + useColocatedData() ? AffinityKey.class : Integer.class, Cancel.class, + useColocatedData() ? AffinityKey.class : Integer.class, Exec.class ); - else if (mode == CacheMode.REPLICATED) - cc.setIndexedTypes( - Integer.class, Exec.class - ); - else - throw new IllegalStateException("mode: " + mode); + } } /** {@inheritDoc} */ @@ -218,11 +216,12 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest { int price = 1000 + rootOrderId; int latsMkt = 3000 + rootOrderId; - Exec exec = new Exec(rootOrderId, dates.get(rootOrderId % dates.size()), execShares, price, latsMkt); + Exec exec = new Exec(idGen.incrementAndGet(), rootOrderId, + dates.get(rootOrderId % dates.size()), execShares, price, latsMkt); add(exec); - rCache.put(exec.rootOrderId, exec); + pCache.put(exec.key(useColocatedData()), exec); insertInDb(exec); } @@ -239,13 +238,20 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest { "from \"part\".ReplaceOrder"); compareQueryRes0("select _key, _val, id, date, orderId, parentAlgo from \"part\".OrderParams\n"); compareQueryRes0("select _key, _val, id, date, refOrderId from \"part\".Cancel\n"); - compareQueryRes0(rCache, "select _key, _val, date, rootOrderId, execShares, price, lastMkt from \"repl\".Exec\n"); + compareQueryRes0("select _key, _val, date, rootOrderId, execShares, price, lastMkt from \"part\".Exec\n"); } /** * @throws Exception If failed. */ public void testBigQuery() throws Exception { + X.println(); + X.println(bigQry); + X.println(); + + X.println("Plan: " + pCache.query(new SqlFieldsQuery("EXPLAIN " + bigQry) + .setDistributedJoins(distributedJoins())).getAll()); + List<List<?>> res = compareQueryRes0(pCache, bigQry, distributedJoins(), new Object[0], Ordering.RANDOM); assertTrue(!res.isEmpty()); // Ensure we set good testing data at database. @@ -301,9 +307,9 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest { " refOrderId int" + " )"); - st.execute("create table \"repl\".Exec" + + st.execute("create table \"part\".Exec" + " (" + - " _key int not null," + + " _key " + keyType + " not null," + " _val other not null," + " rootOrderId int unique," + " date Date, " + @@ -414,11 +420,11 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest { */ private void insertInDb(Exec o) throws SQLException { try(PreparedStatement st = conn.prepareStatement( - "insert into \"repl\".Exec (_key, _val, date, rootOrderId, execShares, price, lastMkt) " + + "insert into \"part\".Exec (_key, _val, date, rootOrderId, execShares, price, lastMkt) " + "values(?, ?, ?, ?, ?, ?, ?)")) { int i = 0; - st.setObject(++i, o.rootOrderId); + st.setObject(++i, o.key(useColocatedData())); st.setObject(++i, o); st.setObject(++i, o.date); st.setObject(++i, o.rootOrderId); @@ -654,8 +660,12 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest { * Execute information about root query. */ static class Exec implements Serializable { - /** Primary key. */ + /** */ @QuerySqlField + private int id; + + /** */ + @QuerySqlField(index = true) private int rootOrderId; /** Date */ @@ -675,13 +685,15 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest { private int lastMkt; /** + * @param id ID. * @param rootOrderId Root order id. * @param date Date. * @param execShares Execute shares. * @param price Price. * @param lastMkt Last mkt. */ - Exec(int rootOrderId, Date date, int execShares, int price, int lastMkt) { + Exec(int id, int rootOrderId, Date date, int execShares, int price, int lastMkt) { + this.id = id; this.rootOrderId = rootOrderId; this.date = date; this.execShares = execShares; @@ -691,12 +703,16 @@ public class H2CompareBigQueryTest extends AbstractH2CompareQueryTest { /** {@inheritDoc} */ @Override public boolean equals(Object o) { - return this == o || o instanceof Exec && rootOrderId == ((Exec)o).rootOrderId; + return this == o || o instanceof Exec && id == ((Exec)o).id; } /** {@inheritDoc} */ @Override public int hashCode() { - return rootOrderId; + return id; + } + + public Object key(boolean useColocatedData) { + return useColocatedData ? new AffinityKey<>(id, rootOrderId) : id; } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/bf337bc5/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql b/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql index efa768c..8d42d44 100644 --- a/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql +++ b/modules/indexing/src/test/resources/org/apache/ignite/internal/processors/query/h2/sql/bigQuery.sql @@ -46,7 +46,7 @@ inner join ( select e.date, e.rootOrderId as eRootOrderId, e.rootOrderId, sum(e.execShares) as execShares, sum(e.execShares*e.price)/sum(e.execShares) as price, case when min(e.lastMkt) = max(e.lastMkt) then min(e.lastMkt) else min('XOFF') end as execMeet - from "repl".Exec e + from "part".Exec e group by e.date, e.rootOrderId ) oep on (cop.date = oep.date and cop.custOrderId = oep.eRootOrderId) left outer join (
