Repository: tajo Updated Branches: refs/heads/master 5982cf1ff -> fd6a95180
TAJO-1752: NameResolver cannot find nested records properly. Closes #686 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/fd6a9518 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/fd6a9518 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/fd6a9518 Branch: refs/heads/master Commit: fd6a95180ecfa05df7cd0b1a8a957057c9788ad6 Parents: 5982cf1 Author: Jihoon Son <[email protected]> Authored: Wed Aug 12 18:37:57 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Wed Aug 12 18:37:57 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 ++ .../java/org/apache/tajo/catalog/NestedPathUtil.java | 2 +- .../main/java/org/apache/tajo/util/StringUtils.java | 3 ++- .../tajo/engine/query/TestSelectNestedRecord.java | 10 ++++++++++ .../TestSelectNestedRecord/sample2/sample2.json | 1 + .../queries/TestSelectNestedRecord/sample2_ddl.sql | 10 +++++++--- .../queries/TestSelectNestedRecord/testSelect3.sql | 1 + .../results/TestSelectNestedRecord/testSelect3.result | 3 +++ .../apache/tajo/plan/nameresolver/NameResolver.java | 13 ++++++++----- 9 files changed, 35 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index f572ebc..c7a8fde 100644 --- a/CHANGES +++ b/CHANGES @@ -216,6 +216,8 @@ Release 0.11.0 - unreleased BUG FIXES + TAJO-1752: NameResolver cannot find nested records properly. (jihoon) + TAJO-1732: During filter push down phase, join conditions are not set properly. (jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java index dec9a55..9c750e8 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java @@ -79,7 +79,7 @@ public class NestedPathUtil { } if (found != null) { - if (found.getDataType().getType() == Type.RECORD) { + if (found.getDataType().getType() == Type.RECORD && paths.length > depth + 1) { return lookupColumnInternal(found, paths, depth + 1); } else { return found; http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java index 61dc855..7a9f0c3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/StringUtils.java @@ -432,7 +432,8 @@ public class StringUtils { public static String join(Object[] objects, String delimiter, int startIndex, int length) { boolean first = true; StringBuilder sb = new StringBuilder(); - for(int i = startIndex; i + startIndex < length; i++) { + int endIndex = startIndex + length; + for(int i = startIndex; i + startIndex < endIndex; i++) { if (first) { first = false; } else { http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java index ffd6f08..41df0fa 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectNestedRecord.java @@ -60,6 +60,16 @@ public class TestSelectNestedRecord extends QueryTestCaseBase { } @Test + public final void testSelect3() throws Exception { + List<String> tables = executeDDL("sample2_ddl.sql", "sample2", "sample5"); + assertEquals(TUtil.newList("sample5"), tables); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test public final void testNestedFieldAsGroupbyKey1() throws Exception { List<String> tables = executeDDL("tweets_ddl.sql", "tweets", "tweets"); assertEquals(TUtil.newList("tweets"), tables); http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json new file mode 100644 index 0000000..20bab62 --- /dev/null +++ b/tajo-core/src/test/resources/dataset/TestSelectNestedRecord/sample2/sample2.json @@ -0,0 +1 @@ +{ "glossary": { "title": "example glossary", "GlossDiv": { "title": "S", "GlossList": { "GlossEntry": { "ID": "SGML", "SortAs": "SGML", "GlossTerm": "Standard Generalized Markup Language", "Acronym": "SGML", "Abbrev": "ISO 8879:1986", "GlossDef": { "para": "A meta-markup language, used to create markup languages such as DocBook.", "GlossSeeAlso": ["GML", "XML"] }, "GlossSee": "markup" } } } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql index 9537c3e..bc2d6e2 100644 --- a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/sample2_ddl.sql @@ -2,18 +2,22 @@ CREATE EXTERNAL TABLE ${0} ( glossary RECORD ( title TEXT, "GlossDiv" RECORD ( - "GlossEntry" RECORD ( + title TEXT, + "GlossList" RECORD ( + "GlossEntry" RECORD ( "ID" TEXT, "SortAs" TEXT, "GlossTerm" TEXT, "Acronym" TEXT, "Abbrev" TEXT, "GlossDef" RECORD ( - para TEXT + para TEXT, + "GlossSeeAlso" TEXT ), "GlossSee" TEXT + ) ) ) ) -) USING JSON LOCATION ${path}; \ No newline at end of file +) USING JSON LOCATION ${table.path}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql new file mode 100644 index 0000000..dfc79e9 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectNestedRecord/testSelect3.sql @@ -0,0 +1 @@ +select glossary.title, glossary."GlossDiv"."title" from sample5; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result new file mode 100644 index 0000000..47e3fe5 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectNestedRecord/testSelect3.result @@ -0,0 +1,3 @@ +glossary/title,glossary/GlossDiv/title +------------------------------- +example glossary,S \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/fd6a9518/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index a6f3d35..c346ce9 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -24,6 +24,7 @@ import com.google.common.collect.Maps; import org.apache.tajo.algebra.ColumnReferenceExpr; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.NestedPathUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.exception.AmbiguousTableException; import org.apache.tajo.catalog.exception.UndefinedColumnException; @@ -346,8 +347,9 @@ public abstract class NameResolver { } // column.nested_fieldX... - if (guessedRelations.size() == 0 && qualifierParts.length == 1) { - Collection<RelationNode> rels = lookupTableByColumns(block, qualifierParts[0]); + if (guessedRelations.size() == 0 && qualifierParts.length > 0) { + Collection<RelationNode> rels = lookupTableByColumns(block, StringUtils.join(qualifierParts, + NestedPathUtil.PATH_DELIMITER, 0)); if (rels.size() > 1) { throw new AmbiguousColumnException(columnRef.getCanonicalName()); @@ -376,12 +378,13 @@ public abstract class NameResolver { columnName = qualifierParts[columnNamePosition]; // if qualifierParts include nested field names - if (qualifierParts.length > columnNamePosition) { - columnName += StringUtils.join(qualifierParts, "/", columnNamePosition + 1, qualifierParts.length); + if (qualifierParts.length > columnNamePosition + 1) { + columnName += NestedPathUtil.PATH_DELIMITER + StringUtils.join(qualifierParts, NestedPathUtil.PATH_DELIMITER, + columnNamePosition + 1, qualifierParts.length); } // columnRef always has a leaf field name. - columnName += "/" + columnRef.getName(); + columnName += NestedPathUtil.PATH_DELIMITER + columnRef.getName(); } return new Pair<String, String>(qualifier, columnName);
