Repository: tajo Updated Branches: refs/heads/master 5853b47b4 -> 9c14dc3ad
TAJO-1991: Tablespace::getVolume should take filter predication. Closes #901 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/9c14dc3a Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/9c14dc3a Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/9c14dc3a Branch: refs/heads/master Commit: 9c14dc3ad794ace2d2ab46c0b0185f265cc703bf Parents: 5853b47 Author: Hyunsik Choi <[email protected]> Authored: Wed Jan 6 21:59:31 2016 -0800 Committer: Hyunsik Choi <[email protected]> Committed: Wed Jan 6 21:59:31 2016 -0800 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/storage/StorageService.java | 42 ------------------- .../org/apache/tajo/plan/LogicalOptimizer.java | 3 +- .../org/apache/tajo/plan/LogicalPlanner.java | 1 - .../org/apache/tajo/plan/StorageService.java | 44 ++++++++++++++++++++ .../rewrite/BaseLogicalPlanRewriteEngine.java | 2 +- .../rewrite/LogicalPlanRewriteRuleContext.java | 2 +- .../plan/rewrite/TableStatUpdateRewriter.java | 12 +++--- .../org/apache/tajo/storage/Tablespace.java | 3 +- .../apache/tajo/storage/TablespaceManager.java | 7 +++- .../tajo/storage/hbase/HBaseTablespace.java | 2 +- .../org/apache/tajo/storage/FileTablespace.java | 2 +- tajo-storage/tajo-storage-jdbc/pom.xml | 1 - .../tajo/storage/jdbc/JdbcTablespace.java | 3 +- 14 files changed, 67 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 2029fe9..7ca2377 100644 --- a/CHANGES +++ b/CHANGES @@ -8,6 +8,8 @@ Release 0.12.0 - unreleased IMPROVEMENT + TAJO-1991: Tablespace::getVolume should take filter predication. (hyunsik) + TAJO-1944: Support text resultset for REST (DaeMyung) TAJO-1950: Query master uses too much memory during range shuffle. (jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java deleted file mode 100644 index 0c3c031..0000000 --- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.tajo.exception.UnsupportedException; - -import javax.annotation.Nullable; -import java.net.URI; - -/** - * TablespaceManager interface for loosely coupled usages - */ -public interface StorageService { - - /** - * Get Table URI - * - * @param spaceName Tablespace name. If it is null, the default space will be used - * @param databaseName Database name - * @param tableName Table name - * @return Table URI - */ - URI getTableURI(@Nullable String spaceName, String databaseName, String tableName); - - long getTableVolumn(URI uri) throws UnsupportedException; -} http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index 23f8584..65d1759 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -30,8 +30,8 @@ import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.plan.expr.*; import org.apache.tajo.exception.TajoException; +import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.joinorder.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteEngine; @@ -39,7 +39,6 @@ import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleProvider; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; -import org.apache.tajo.storage.StorageService; import org.apache.tajo.util.ReflectionUtil; import org.apache.tajo.util.graph.DirectedGraphCursor; http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 50ab2e0..190e12b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -48,7 +48,6 @@ import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.storage.StorageService; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; import org.apache.tajo.util.StringUtils; http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java new file mode 100644 index 0000000..10d11f0 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan; + +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.expr.EvalNode; + +import javax.annotation.Nullable; +import java.net.URI; +import java.util.Optional; + +/** + * TablespaceManager interface for loosely coupled usages + */ +public interface StorageService { + + /** + * Get Table URI + * + * @param spaceName Tablespace name. If it is null, the default space will be used + * @param databaseName Database name + * @param tableName Table name + * @return Table URI + */ + URI getTableURI(@Nullable String spaceName, String databaseName, String tableName); + + long getTableVolumn(URI uri, Optional<EvalNode> filter) throws UnsupportedException; +} http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java index f38b202..21a0905 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteEngine.java @@ -22,7 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.storage.StorageService; +import org.apache.tajo.plan.StorageService; import java.util.LinkedHashMap; import java.util.Map; http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java index a899961..aa7308a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java @@ -21,7 +21,7 @@ package org.apache.tajo.plan.rewrite; import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.plan.LogicalPlan; -import org.apache.tajo.storage.StorageService; +import org.apache.tajo.plan.StorageService; public class LogicalPlanRewriteRuleContext { http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java index d4c4363..3683f60 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java @@ -27,11 +27,13 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.StorageService; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; -import org.apache.tajo.storage.StorageService; +import java.util.Optional; import java.util.Stack; public class TableStatUpdateRewriter implements LogicalPlanRewriteRule { @@ -75,15 +77,15 @@ public class TableStatUpdateRewriter implements LogicalPlanRewriteRule { final TableDesc table = scanNode.getTableDesc(); if (!isVirtual(table)) { - final TableStats stats = getTableStat(table); final long tableSize = stats.getNumBytes(); + final Optional<EvalNode> filter = scanNode.hasQual() ? Optional.of(scanNode.getQual()) : Optional.empty(); // If USE_TABLE_VOLUME is set, we will update the table volume through a storage handler. // In addition, if the table size is zero, we will update too. // It is a good workaround to avoid suboptimal join orders without cheap cost. if (conf.getBool(SessionVars.USE_TABLE_VOLUME) || tableSize == 0) { - table.getStats().setNumBytes(getTableVolume(table)); + table.getStats().setNumBytes(getTableVolume(table, filter)); } } @@ -105,10 +107,10 @@ public class TableStatUpdateRewriter implements LogicalPlanRewriteRule { return table.getMeta().getDataFormat().equals("SYSTEM"); } - private long getTableVolume(TableDesc table) { + private long getTableVolume(TableDesc table, Optional<EvalNode> filter) { try { if (table.getStats() != null) { - return storage.getTableVolumn(table.getUri()); + return storage.getTableVolumn(table.getUri(), filter); } } catch (UnsupportedException t) { LOG.warn(table.getName() + " does not support Tablespace::getTableVolume()"); http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java index 81fbbc0..6c97754 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.net.URI; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Set; /** @@ -98,7 +99,7 @@ public abstract class Tablespace { return name + "=" + uri.toString(); } - public abstract long getTableVolume(URI uri) throws UnsupportedException; + public abstract long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException; /** * if {@link StorageProperty#isArbitraryPathAllowed} is true, http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java index b1e3275..12e283f 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java @@ -36,6 +36,8 @@ import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UndefinedTablespaceException; import org.apache.tajo.exception.UndefinedTablespaceHandlerException; import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.plan.StorageService; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.JavaResourceUtil; import org.apache.tajo.util.Pair; @@ -433,8 +435,9 @@ public class TablespaceManager implements StorageService { } @Override - public long getTableVolumn(URI tableUri) throws UnsupportedException { - return get(tableUri).getTableVolume(tableUri); + public long getTableVolumn(URI tableUri, Optional<EvalNode> filter) + throws UnsupportedException { + return get(tableUri).getTableVolume(tableUri, filter); } public static Iterable<Tablespace> getAllTablespaces() { http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java index c79de33..3541c50 100644 --- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java +++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java @@ -100,7 +100,7 @@ public class HBaseTablespace extends Tablespace { } @Override - public long getTableVolume(URI uri) throws UnsupportedException { + public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException { throw new UnsupportedException(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index d95852c..ee4512b 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -125,7 +125,7 @@ public class FileTablespace extends Tablespace { } @Override - public long getTableVolume(URI uri) throws UnsupportedException { + public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException { Path path = new Path(uri); ContentSummary summary; try { http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml index 61481c7..804cad4 100644 --- a/tajo-storage/tajo-storage-jdbc/pom.xml +++ b/tajo-storage/tajo-storage-jdbc/pom.xml @@ -193,7 +193,6 @@ <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> </dependency> - </dependencies> <profiles> http://git-wip-us.apache.org/repos/asf/tajo/blob/9c14dc3a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java index 7489307..1f7f299 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java @@ -47,6 +47,7 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Properties; /** @@ -109,7 +110,7 @@ public abstract class JdbcTablespace extends Tablespace { } @Override - public long getTableVolume(URI uri) throws UnsupportedException { + public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException { throw new UnsupportedException(); }
