Repository: tajo Updated Branches: refs/heads/branch-0.11.0 8b318f6ea -> 1aea89c7f
TAJO-1801: Table name is not unique of tableDescMap in QueryMasterTask. Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/1aea89c7 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/1aea89c7 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/1aea89c7 Branch: refs/heads/branch-0.11.0 Commit: 1aea89c7fdc799912f7fdd98837e983736036e9b Parents: 8b318f6 Author: Jihoon Son <[email protected]> Authored: Fri Aug 28 01:39:02 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Aug 28 01:39:02 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 +++ .../main/java/org/apache/tajo/querymaster/Query.java | 12 ++++-------- .../org/apache/tajo/querymaster/QueryMasterTask.java | 12 ++++++------ .../org/apache/tajo/querymaster/Repartitioner.java | 6 +++--- .../main/java/org/apache/tajo/querymaster/Stage.java | 14 ++++++-------- 5 files changed, 22 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/1aea89c7/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 9fc3de0..751b61f 100644 --- a/CHANGES +++ b/CHANGES @@ -232,6 +232,9 @@ Release 0.11.0 - unreleased BUG FIXES + TAJO-1801: Table name is not unique of tableDescMap in QueryMasterTask. + (jihoon) + TAJO-1798: Dynamic partitioning occasionally fails. (jaehwa) TAJO-1799: Fix incorrect event handler when kill-query failed. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/1aea89c7/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 9560353..8bb3845 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -35,26 +35,22 @@ import org.apache.tajo.QueryVars; import org.apache.tajo.SessionVars; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; -import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; import org.apache.tajo.engine.planner.global.ExecutionQueue; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.exception.TajoException; -import org.apache.tajo.exception.TajoInternalError; -import org.apache.tajo.plan.logical.*; import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.master.event.*; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.StorageConstants; -import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.history.QueryHistory; import org.apache.tajo.util.history.StageHistory; http://git-wip-us.apache.org/repos/asf/tajo/blob/1aea89c7/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 52e0a96..1313dad 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -91,7 +91,7 @@ public class QueryMasterTask extends CompositeService { private final long querySubmitTime; - private Map<String, TableDesc> tableDescMap = new HashMap<String, TableDesc>(); + private final Map<Integer, TableDesc> tableDescMap = new HashMap<>(); private TajoConf systemConf; @@ -332,7 +332,7 @@ public class QueryMasterTask extends CompositeService { if (scanNodes != null) { for (LogicalNode eachScanNode : scanNodes) { ScanNode scanNode = (ScanNode) eachScanNode; - tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); + tableDescMap.put(scanNode.getPID(), scanNode.getTableDesc()); } } @@ -340,7 +340,7 @@ public class QueryMasterTask extends CompositeService { if (scanNodes != null) { for (LogicalNode eachScanNode : scanNodes) { ScanNode scanNode = (ScanNode) eachScanNode; - tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); + tableDescMap.put(scanNode.getPID(), scanNode.getTableDesc()); } } @@ -348,7 +348,7 @@ public class QueryMasterTask extends CompositeService { if (scanNodes != null) { for (LogicalNode eachScanNode : scanNodes) { ScanNode scanNode = (ScanNode) eachScanNode; - tableDescMap.put(scanNode.getCanonicalName(), scanNode.getTableDesc()); + tableDescMap.put(scanNode.getPID(), scanNode.getTableDesc()); } } } @@ -533,8 +533,8 @@ public class QueryMasterTask extends CompositeService { return query.getStage(id); } - public Map<String, TableDesc> getTableDescMap() { - return tableDescMap; + public TableDesc getTableDesc(ScanNode scanNode) { + return tableDescMap.get(scanNode.getPID()); } public float getProgress() { http://git-wip-us.apache.org/repos/asf/tajo/blob/1aea89c7/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java index 0d5880e..c4fc645 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Repartitioner.java @@ -88,7 +88,7 @@ public class Repartitioner { // initialize variables from the child operators for (int i = 0; i < scans.length; i++) { - TableDesc tableDesc = masterContext.getTableDescMap().get(scans[i].getCanonicalName()); + TableDesc tableDesc = masterContext.getTableDesc(scans[i]); if (tableDesc == null) { // if it is a real table stored on storage if (execBlock.getUnionScanMap() != null && !execBlock.getUnionScanMap().isEmpty()) { @@ -376,7 +376,7 @@ public class Repartitioner { for (ScanNode eachScan: broadcastScans) { Path[] partitionScanPaths = null; - TableDesc tableDesc = masterContext.getTableDescMap().get(eachScan.getCanonicalName()); + TableDesc tableDesc = masterContext.getTableDesc(eachScan); Tablespace space = TablespaceManager.get(tableDesc.getUri()).get(); if (eachScan.getType() == NodeType.PARTITIONS_SCAN) { @@ -498,7 +498,7 @@ public class Repartitioner { List<Fragment> broadcastFragments = new ArrayList<Fragment>(); for (int i = 0; i < scans.length; i++) { ScanNode scan = scans[i]; - TableDesc desc = stage.getContext().getTableDescMap().get(scan.getCanonicalName()); + TableDesc desc = stage.getContext().getTableDesc(scan); TableMeta meta = desc.getMeta(); Collection<Fragment> scanFragments; http://git-wip-us.apache.org/repos/asf/tajo/blob/1aea89c7/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java index f6c9cdb..12e4366 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Stage.java @@ -31,7 +31,6 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.catalog.statistics.StatisticsUtil; @@ -44,12 +43,12 @@ import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.exception.TajoException; import org.apache.tajo.ipc.TajoWorkerProtocol; -import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage; -import org.apache.tajo.plan.serder.PlanProto.EnforceProperty; import org.apache.tajo.master.TaskState; import org.apache.tajo.master.event.*; import org.apache.tajo.master.event.TaskAttemptToSchedulerEvent.TaskAttemptScheduleContext; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.serder.PlanProto.DistinctGroupbyEnforcer.MultipleAggregationStage; +import org.apache.tajo.plan.serder.PlanProto.EnforceProperty; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.querymaster.Task.IntermediateEntry; import org.apache.tajo.rpc.AsyncRpcClient; @@ -57,8 +56,8 @@ import org.apache.tajo.rpc.NullCallback; import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.storage.FileTablespace; -import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.Tablespace; +import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; @@ -76,9 +75,9 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import static org.apache.tajo.ResourceProtos.*; import static org.apache.tajo.conf.TajoConf.ConfVars; import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; -import static org.apache.tajo.ResourceProtos.*; /** @@ -1058,12 +1057,11 @@ public class Stage implements EventHandler<StageEvent> { public static long getInputVolume(MasterPlan masterPlan, QueryMasterTask.QueryMasterTaskContext context, ExecutionBlock execBlock) { - Map<String, TableDesc> tableMap = context.getTableDescMap(); if (masterPlan.isLeaf(execBlock)) { ScanNode[] outerScans = execBlock.getScanNodes(); long maxVolume = 0; for (ScanNode eachScanNode: outerScans) { - TableStats stat = tableMap.get(eachScanNode.getCanonicalName()).getStats(); + TableStats stat = context.getTableDesc(eachScanNode).getStats(); if (stat.getNumBytes() > maxVolume) { maxVolume = stat.getNumBytes(); } @@ -1089,7 +1087,7 @@ public class Stage implements EventHandler<StageEvent> { ScanNode[] scans = execBlock.getScanNodes(); Preconditions.checkArgument(scans.length == 1, "Must be Scan Query"); ScanNode scan = scans[0]; - TableDesc table = stage.context.getTableDescMap().get(scan.getCanonicalName()); + TableDesc table = stage.context.getTableDesc(scan); Collection<Fragment> fragments; Tablespace tablespace = TablespaceManager.get(scan.getTableDesc().getUri()).get();
