merged master
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/c4acebd5 Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/c4acebd5 Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/c4acebd5 Branch: refs/heads/master Commit: c4acebd5d3911bc208c3fd919a8cf7894b371c00 Parents: a07b8fe 2fe6a2b Author: JoÌrg Unbehauen <jo...@unbehauen.net> Authored: Fri Aug 4 15:31:21 2017 +0200 Committer: JoÌrg Unbehauen <jo...@unbehauen.net> Committed: Fri Aug 4 15:31:21 2017 +0200 ---------------------------------------------------------------------- CHANGES.md | 2 + README.md | 78 +++++------ .../org/apache/metamodel/MetaModelHelper.java | 138 +++++++++++-------- .../factory/DataContextPropertiesImpl.java | 3 + .../apache/metamodel/MetaModelHelperTest.java | 83 +++++++++-- jdbc/pom.xml | 4 + .../metamodel/jdbc/JdbcMetadataLoader.java | 92 ++++++++++--- .../jdbc/dialects/DefaultQueryRewriter.java | 24 ++-- .../jdbc/dialects/HsqldbQueryRewriter.java | 13 ++ .../apache/metamodel/jdbc/H2databaseTest.java | 104 +++++++++----- .../org/apache/metamodel/jdbc/HsqldbTest.java | 112 +++++++++++---- .../metamodel/jdbc/MultiJDBCDataSetTest.java | 132 ++++++++++++++++++ .../apache/metamodel/pojo/PojoDataContext.java | 4 +- .../metamodel/pojo/PojoDataContextFactory.java | 71 ++++++++++ ....apache.metamodel.factory.DataContextFactory | 1 + pom.xml | 84 ++++++++++- spring/pom.xml | 12 -- 17 files changed, 732 insertions(+), 225 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metamodel/blob/c4acebd5/core/src/main/java/org/apache/metamodel/MetaModelHelper.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/metamodel/MetaModelHelper.java index 4f0a8dd,a2681da..f23c98e --- a/core/src/main/java/org/apache/metamodel/MetaModelHelper.java +++ b/core/src/main/java/org/apache/metamodel/MetaModelHelper.java @@@ -18,19 -18,10 +18,12 @@@ */ package org.apache.metamodel; - import java.util.ArrayList; - import java.util.Arrays; - import java.util.Collection; - import java.util.Collections; - import java.util.Comparator; - import java.util.HashMap; - import java.util.HashSet; - import java.util.List; - import java.util.Map; + import java.util.*; import java.util.Map.Entry; import java.util.stream.Collectors; +import java.util.stream.Stream; + + import org.apache.metamodel.data.CachingDataSetHeader; import org.apache.metamodel.data.DataSet; import org.apache.metamodel.data.DataSetHeader; @@@ -184,67 -180,86 +182,87 @@@ public final class MetaModelHelper if (fromDataSets.length == 1) { return getFiltered(fromDataSets[0], whereItems); } + // do a nested loop join, no matter what + Iterator<DataSet> dsIter = Arrays.asList(fromDataSets).iterator(); - List<SelectItem> selectItems = new ArrayList<SelectItem>(); - for (DataSet dataSet : fromDataSets) { - for (int i = 0; i < dataSet.getSelectItems().size(); i++) { - SelectItem item = dataSet.getSelectItems().get(i); - selectItems.add(item); - } - } + - int selectItemOffset = 0; - List<Object[]> data = new ArrayList<Object[]>(); - for (int fromDataSetIndex = 0; fromDataSetIndex < fromDataSets.length; fromDataSetIndex++) { - DataSet fromDataSet = fromDataSets[fromDataSetIndex]; - List<SelectItem> fromSelectItems = fromDataSet.getSelectItems(); - if (fromDataSetIndex == 0) { - while (fromDataSet.next()) { - Object[] values = fromDataSet.getRow().getValues(); - Object[] row = new Object[selectItems.size()]; - System.arraycopy(values, 0, row, selectItemOffset, values.length); - data.add(row); - } - fromDataSet.close(); - } else { - List<Object[]> fromDataRows = new ArrayList<Object[]>(); - while (fromDataSet.next()) { - fromDataRows.add(fromDataSet.getRow().getValues()); - } - fromDataSet.close(); - for (int i = 0; i < data.size(); i = i + fromDataRows.size()) { - Object[] originalRow = data.get(i); - data.remove(i); - for (int j = 0; j < fromDataRows.size(); j++) { - Object[] newRow = fromDataRows.get(j); - System.arraycopy(newRow, 0, originalRow, selectItemOffset, newRow.length); - data.add(i + j, originalRow.clone()); - } - } - } - selectItemOffset += fromSelectItems.size(); - } + DataSet joined = dsIter.next(); + + while (dsIter.hasNext()) { + joined = nestedLoopJoin(dsIter.next(), joined, (whereItems)); - if (data.isEmpty()) { - return new EmptyDataSet(selectItems); } - final DataSetHeader header = new CachingDataSetHeader(selectItems); - final List<Row> rows = new ArrayList<Row>(data.size()); - for (Object[] objects : data) { - rows.add(new DefaultRow(header, objects, null)); + return joined; + + } + + /** + * Executes a simple nested loop join. The innerLoopDs will be copied in an + * in-memory dataset. + * + */ + public static InMemoryDataSet nestedLoopJoin(DataSet innerLoopDs, DataSet outerLoopDs, + Iterable<FilterItem> filtersIterable) { + + List<FilterItem> filters = new ArrayList<>(); + for (FilterItem fi : filtersIterable) { + filters.add(fi); } + List<Row> innerRows = innerLoopDs.toRows(); + - List<SelectItem> allItems = new ArrayList<>(Arrays.asList(outerLoopDs.getSelectItems())); - allItems.addAll(Arrays.asList(innerLoopDs.getSelectItems())); ++ List<SelectItem> allItems = new ArrayList<>(outerLoopDs.getSelectItems()); ++ allItems.addAll(innerLoopDs.getSelectItems()); - DataSet result = new InMemoryDataSet(header, rows); - if (whereItems != null) { - DataSet filteredResult = getFiltered(result, whereItems); - result = filteredResult; + Set<FilterItem> applicableFilters = applicableFilters(filters, allItems); + + DataSetHeader jointHeader = new CachingDataSetHeader(allItems); + + List<Row> resultRows = new ArrayList<>(); + for (Row outerRow : outerLoopDs) { + for (Row innerRow : innerRows) { + + Object[] joinedRowObjects = new Object[outerRow.getValues().length + innerRow.getValues().length]; + + System.arraycopy(outerRow.getValues(), 0, joinedRowObjects, 0, outerRow.getValues().length); + System.arraycopy(innerRow.getValues(), 0, joinedRowObjects, outerRow.getValues().length, innerRow + .getValues().length); + + Row joinedRow = new DefaultRow(jointHeader, joinedRowObjects); + + if (applicableFilters.isEmpty() || applicableFilters.stream().allMatch(fi -> fi.accept(joinedRow))) { + resultRows.add(joinedRow); + } + } } - return result; + + return new InMemoryDataSet(jointHeader, resultRows); } - public static DataSet getCarthesianProduct(DataSet[] fromDataSets, FilterItem... filterItems) { - return getCarthesianProduct(fromDataSets, Arrays.asList(filterItems)); + /** + * Filters the FilterItems such that only the FilterItems are returned, + * which contain SelectItems that are contained in selectItemList + * + * @param filters + * @param selectItemList + * @return + */ + private static Set<FilterItem> applicableFilters(Collection<FilterItem> filters, + Collection<SelectItem> selectItemList) { + + Set<SelectItem> items = new HashSet<SelectItem>(selectItemList); + + return filters.stream().filter(fi -> { + Collection<SelectItem> fiSelectItems = new ArrayList<>(); + fiSelectItems.add(fi.getSelectItem()); + Object operand = fi.getOperand(); + if (operand instanceof SelectItem) { + fiSelectItems.add((SelectItem) operand); + } + + return items.containsAll(fiSelectItems); + + }).collect(Collectors.toSet()); } public static DataSet getFiltered(DataSet dataSet, Iterable<FilterItem> filterItems) { http://git-wip-us.apache.org/repos/asf/metamodel/blob/c4acebd5/core/src/test/java/org/apache/metamodel/MetaModelHelperTest.java ---------------------------------------------------------------------- diff --cc core/src/test/java/org/apache/metamodel/MetaModelHelperTest.java index 84a09b2,a84cef1..7196bf4 --- a/core/src/test/java/org/apache/metamodel/MetaModelHelperTest.java +++ b/core/src/test/java/org/apache/metamodel/MetaModelHelperTest.java @@@ -116,21 -115,19 +116,21 @@@ public class MetaModelHelperTest extend public void testSimpleCarthesianProduct() throws Exception { DataSet dataSet = MetaModelHelper.getCarthesianProduct(createDataSet1(), createDataSet2()); + List<String> results = new ArrayList<String>(); ++ + while (dataSet.next()) { + results.add(dataSet.getRow().toString()); + } - assertEquals(2, dataSet.getSelectItems().length); + assertEquals(2, dataSet.getSelectItems().size()); - assertTrue(dataSet.next()); - assertEquals("Row[values=[f, b]]", dataSet.getRow().toString()); - assertTrue(dataSet.next()); - assertEquals("Row[values=[f, a]]", dataSet.getRow().toString()); - assertTrue(dataSet.next()); - assertTrue(dataSet.next()); - assertTrue(dataSet.next()); - assertTrue(dataSet.next()); - assertTrue(dataSet.next()); - assertTrue(dataSet.next()); - assertTrue(dataSet.next()); - assertEquals("Row[values=[o, r]]", dataSet.getRow().toString()); - assertFalse(dataSet.next()); + assertEquals(9, results.size()); + assertTrue(results.contains("Row[values=[f, b]]")); + assertTrue(results.contains("Row[values=[f, a]]")); + assertTrue(results.contains("Row[values=[f, r]]")); + assertTrue(results.contains("Row[values=[o, b]]")); + assertTrue(results.contains("Row[values=[o, a]]")); + assertTrue(results.contains("Row[values=[o, r]]")); ++ } public void testTripleCarthesianProduct() throws Exception { @@@ -188,8 -185,8 +188,10 @@@ data1.add(new Object[] { "f" }); data1.add(new Object[] { "o" }); data1.add(new Object[] { "o" }); - DataSet dataSet1 = createDataSet(new SelectItem[] { new SelectItem(new MutableColumn("foo", - ColumnType.VARCHAR)) }, data1); ++ + DataSet dataSet1 = createDataSet( + Lists.newArrayList( new SelectItem(new MutableColumn("foo", ColumnType.VARCHAR)) ), data1); ++ return dataSet1; } @@@ -206,8 -203,8 +208,10 @@@ List<Object[]> data3 = new ArrayList<Object[]>(); data3.add(new Object[] { "w00p", true }); data3.add(new Object[] { "yippie", false }); - DataSet dataSet3 = createDataSet(new SelectItem[] { new SelectItem("expression", "e"), new SelectItem("webish?", - "w") }, data3); ++ + DataSet dataSet3 = createDataSet(Lists.newArrayList(new SelectItem("expression", "e"), + new SelectItem("webish?", "w") ), data3); ++ return dataSet3; } @@@ -217,6 -214,41 +221,41 @@@ return dataSet4; } + private int bigDataSetSize = 3000; + + /** + * + * @return a big dataset, mocking an employee table + */ + private DataSet createDataSet5() { + List<Object[]> data5 = new ArrayList<Object[]>(); + + for (int i = 0; i < bigDataSetSize; i++) { + data5.add(new Object[] { i, "Person_" + i, bigDataSetSize - (i + 1) }); + } + - DataSet dataSet5 = createDataSet(new SelectItem[] { new SelectItem(new MutableColumn("nr", ColumnType.BIGINT)), ++ DataSet dataSet5 = createDataSet(Lists.newArrayList( new SelectItem(new MutableColumn("nr", ColumnType.BIGINT)), + new SelectItem(new MutableColumn("name", ColumnType.STRING)), new SelectItem(new MutableColumn("dnr", - ColumnType.BIGINT)) }, data5); ++ ColumnType.BIGINT)) ), data5); + return dataSet5; + } + + /** + * + * @return a big dataset, mocking an department table + */ + private DataSet createDataSet6() { + List<Object[]> data6 = new ArrayList<Object[]>(); + + for (int i = 0; i < bigDataSetSize; i++) { + data6.add(new Object[] { i, "Department_" + i }); + } + - DataSet dataSet6 = createDataSet(new SelectItem[] { new SelectItem(new MutableColumn("nr", ColumnType.BIGINT)), - new SelectItem(new MutableColumn("name", ColumnType.STRING)), }, data6); ++ DataSet dataSet6 = createDataSet(Lists.newArrayList(new SelectItem(new MutableColumn("nr", ColumnType.BIGINT)), ++ new SelectItem(new MutableColumn("name", ColumnType.STRING))), data6); + return dataSet6; + } + public void testGetTables() throws Exception { MutableTable table1 = new MutableTable("table1"); MutableTable table2 = new MutableTable("table2"); @@@ -325,4 -357,22 +364,22 @@@ assertEquals("Row[values=[1, 2, null]]", joinedDs.getRow().toString()); assertFalse(joinedDs.next()); } + + public void testCarthesianProductScalability() { + + DataSet employees = createDataSet5(); + DataSet departmens = createDataSet6(); + - FilterItem fi = new FilterItem(employees.getSelectItems()[2], OperatorType.EQUALS_TO, departmens - .getSelectItems()[0]); ++ FilterItem fi = new FilterItem(employees.getSelectItems().get(2), OperatorType.EQUALS_TO, departmens ++ .getSelectItems().get(0)); + + DataSet joined = MetaModelHelper.getCarthesianProduct(new DataSet[] { employees, departmens }, fi); + int count = 0; + while (joined.next()) { + count++; + } + + assertTrue(count == bigDataSetSize); + + } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/c4acebd5/jdbc/src/main/java/org/apache/metamodel/jdbc/JdbcMetadataLoader.java ---------------------------------------------------------------------- diff --cc jdbc/src/main/java/org/apache/metamodel/jdbc/JdbcMetadataLoader.java index 49e242b,2c29405..ce64b87 --- a/jdbc/src/main/java/org/apache/metamodel/jdbc/JdbcMetadataLoader.java +++ b/jdbc/src/main/java/org/apache/metamodel/jdbc/JdbcMetadataLoader.java @@@ -452,9 -468,42 +467,42 @@@ final class JdbcMetadataLoader implemen logger.error("pkColumn={}", pkColumn); logger.error("fkColumn={}", fkColumn); } else { - MutableRelationship.createRelationship(pkColumn, fkColumn); + + if (!relations.containsKey(pkTable)) { + relations.put(pkTable, new HashMap<>()); + } + + // get or init the columns tuple + ColumnsTuple ct = relations.get(pkTable).get(fkTable); + if (Objects.isNull(ct)) { + ct = new ColumnsTuple(); + relations.get(pkTable).put(fkTable, ct); + } + // we can now safely add the columns + ct.getPkCols().add(pkColumn); + ct.getFkCols().add(fkColumn); } } + + relations.values().stream().flatMap(map -> map.values().stream()).forEach(ct -> MutableRelationship - .createRelationship(ct.getPkCols().toArray(new Column[0]), ct.getFkCols().toArray(new Column[0]))); ++ .createRelationship(ct.getPkCols(), ct.getFkCols())); + } + + /** + * Represents the columns of a relationship while it is being built from a + * {@link ResultSet}. + */ + private static class ColumnsTuple { + private final List<Column> pkCols = new ArrayList<>(); + private final List<Column> fkCols = new ArrayList<>(); + + public List<Column> getFkCols() { + return fkCols; + } + + public List<Column> getPkCols() { + return pkCols; + } } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/c4acebd5/jdbc/src/test/java/org/apache/metamodel/jdbc/H2databaseTest.java ---------------------------------------------------------------------- diff --cc jdbc/src/test/java/org/apache/metamodel/jdbc/H2databaseTest.java index e7b5386,d46bc31..18b02de --- a/jdbc/src/test/java/org/apache/metamodel/jdbc/H2databaseTest.java +++ b/jdbc/src/test/java/org/apache/metamodel/jdbc/H2databaseTest.java @@@ -276,14 -281,17 +281,17 @@@ public class H2databaseTest extends Tes @Override public void run(UpdateCallback cb) { - JdbcCreateTableBuilder createTableBuilder = (JdbcCreateTableBuilder) cb.createTable(schema, "test_table"); + JdbcCreateTableBuilder createTableBuilder = (JdbcCreateTableBuilder) cb.createTable(schema, + "test_table"); Table writtenTable = createTableBuilder.withColumn("id").asPrimaryKey().ofType(ColumnType.INTEGER) - .withColumn("name").ofSize(255).ofType(ColumnType.VARCHAR).withColumn("age").ofType(ColumnType.INTEGER) - .execute(); + .withColumn("name").ofSize(255).ofType(ColumnType.VARCHAR).withColumn("age").ofType( + ColumnType.INTEGER).execute(); String sql = createTableBuilder.createSqlStatement(); - assertEquals("CREATE TABLE PUBLIC.test_table (id INTEGER, name VARCHAR(255), age INTEGER, PRIMARY KEY(id))", sql); + assertEquals( + "CREATE TABLE PUBLIC.test_table (id INTEGER, name VARCHAR(255), age INTEGER, PRIMARY KEY(id))", + sql); assertNotNull(writtenTable); - assertEquals("[ID, NAME, AGE]", Arrays.toString(writtenTable.getColumnNames())); + assertEquals("[ID, NAME, AGE]", Arrays.toString(writtenTable.getColumnNames().toArray())); writtenTableRef.set(writtenTable); } @@@ -521,4 -531,32 +531,32 @@@ public void testInterpretationOfNull() throws Exception { JdbcTestTemplates.interpretationOfNulls(conn); } + + public void testCompositeFkRelation() throws Exception { + + try (Statement stmt = conn.createStatement()) { + stmt.execute( + "CREATE TABLE PARENT (P1 INTEGER, P2 INTEGER, P3 INTEGER, P4 INTEGER, PRIMARY KEY (P1,P2, P3, P4))"); + stmt.execute( + "CREATE TABLE CHILD (C1 INTEGER PRIMARY KEY, CP1 INTEGER , CP2 INTEGER, CP3 INTEGER, CP4 INTEGER, FOREIGN KEY (CP1,CP2,CP3,CP4) REFERENCES PARENT(P1,P2,P3,P4))"); + } + + final JdbcDataContext dc = new JdbcDataContext(conn); + + final Schema schema = dc.getDefaultSchema(); + - assertEquals(1, schema.getRelationships().length); ++ assertEquals(1, schema.getRelationships().size()); + - Relationship rel = schema.getRelationships()[0]; ++ Relationship rel = schema.getRelationships().iterator().next(); + - assertEquals("CP1", rel.getForeignColumns()[0].getName()); - assertEquals("CP2", rel.getForeignColumns()[1].getName()); - assertEquals("CP3", rel.getForeignColumns()[2].getName()); - assertEquals("CP4", rel.getForeignColumns()[3].getName()); ++ assertEquals("CP1", rel.getForeignColumns().get(0).getName()); ++ assertEquals("CP2", rel.getForeignColumns().get(1).getName()); ++ assertEquals("CP3", rel.getForeignColumns().get(2).getName()); ++ assertEquals("CP4", rel.getForeignColumns().get(3).getName()); + - assertEquals("P1", rel.getPrimaryColumns()[0].getName()); - assertEquals("P2", rel.getPrimaryColumns()[1].getName()); - assertEquals("P3", rel.getPrimaryColumns()[2].getName()); - assertEquals("P4", rel.getPrimaryColumns()[3].getName()); ++ assertEquals("P1", rel.getPrimaryColumns().get(0).getName()); ++ assertEquals("P2", rel.getPrimaryColumns().get(1).getName()); ++ assertEquals("P3", rel.getPrimaryColumns().get(2).getName()); ++ assertEquals("P4", rel.getPrimaryColumns().get(3).getName()); + } } http://git-wip-us.apache.org/repos/asf/metamodel/blob/c4acebd5/jdbc/src/test/java/org/apache/metamodel/jdbc/HsqldbTest.java ---------------------------------------------------------------------- diff --cc jdbc/src/test/java/org/apache/metamodel/jdbc/HsqldbTest.java index bf64fdd,d9f0587..3d9d268 --- a/jdbc/src/test/java/org/apache/metamodel/jdbc/HsqldbTest.java +++ b/jdbc/src/test/java/org/apache/metamodel/jdbc/HsqldbTest.java @@@ -146,8 -145,8 +145,9 @@@ public class HsqldbTest extends TestCas Table productsTable = schema.getTableByName("PRODUCTS"); Table factTable = schema.getTableByName("ORDERFACT"); - Query q = new Query().from(new FromItem(JoinType.INNER, productsTable.getRelationships(factTable)[0])) - .select(productsTable.getColumns()[0], factTable.getColumns()[0]); + Query q = new Query().from(new FromItem(JoinType.INNER, productsTable.getRelationships(factTable).iterator().next())).select( + productsTable.getColumns().get(0), factTable.getColumns().get(0)); ++ assertEquals( "SELECT \"PRODUCTS\".\"PRODUCTCODE\", \"ORDERFACT\".\"ORDERNUMBER\" FROM PUBLIC.\"PRODUCTS\" INNER JOIN PUBLIC.\"ORDERFACT\" ON \"PRODUCTS\".\"PRODUCTCODE\" = \"ORDERFACT\".\"PRODUCTCODE\"", q.toString()); http://git-wip-us.apache.org/repos/asf/metamodel/blob/c4acebd5/pojo/src/main/java/org/apache/metamodel/pojo/PojoDataContext.java ----------------------------------------------------------------------
