Repository: tajo Updated Branches: refs/heads/index_support 72600c138 -> 761e372ba
TAJO-1231: Implicit table properties in session is not stored in table property. Closes #287 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ab2efce8 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ab2efce8 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ab2efce8 Branch: refs/heads/index_support Commit: ab2efce8fb4015523a9a818f0e03fa57c6366585 Parents: 2a69bcc Author: Hyunsik Choi <[email protected]> Authored: Fri Dec 5 11:43:15 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Fri Dec 5 16:36:12 2014 +0900 ---------------------------------------------------------------------- CHANGES | 7 ++-- .../org/apache/tajo/client/TestTajoClient.java | 37 +++++++++++++++++--- .../org/apache/tajo/plan/LogicalPlanner.java | 20 ++++++++--- .../org/apache/tajo/plan/util/PlannerUtil.java | 17 +++++++++ 4 files changed, 70 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 0c785ce..8ecd795 100644 --- a/CHANGES +++ b/CHANGES @@ -83,8 +83,11 @@ Release 0.9.1 - unreleased BUG FIXES - TAJO-1194: 'INSERT OVERWRITE .. SELECT' does not remove existing data when result is empty. - (jaehwa) + TAJO-1231: Implicit table properties in session are not stored in + table property. (hyunsik) + + TAJO-1194: 'INSERT OVERWRITE .. SELECT' does not remove existing data + when result is empty. (jaehwa) TAJO-1191: Change DateDatum timezone to UTC. (Jaewoong Jung via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java index 616799f..21dd7d5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -680,7 +680,7 @@ public class TestTajoClient { } @Test - public void testSetCvsNull() throws Exception { + public void testNullCharSession() throws Exception { String sql = "select\n" + " c_custkey,\n" + @@ -692,17 +692,44 @@ public class TestTajoClient { " c_custkey,\n" + " orders.o_orderkey;\n"; - TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - Map<String, String> variables = new HashMap<String, String>(); variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T"); client.updateSessionVariables(variables); + TajoResultSet resultDesc = (TajoResultSet)client.executeQueryAndGetResult(sql); + resultDesc.close(); + assertNullCharSessionVar(resultDesc.getTableDesc()); + } + + @Test + public void testNullCharSessionInCTAS() throws Exception { + String sql = + "create table nullcharsession as select\n" + + " c_custkey,\n" + + " orders.o_orderkey,\n" + + " orders.o_orderstatus \n" + + "from\n" + + " orders full outer join customer on c_custkey = o_orderkey\n" + + "order by\n" + + " c_custkey,\n" + + " orders.o_orderkey;\n"; + Map<String, String> variables = new HashMap<String, String>(); + variables.put(SessionVars.NULL_CHAR.keyname(), "\\\\T"); + client.updateSessionVariables(variables); TajoResultSet res = (TajoResultSet)client.executeQueryAndGetResult(sql); + res.close(); + + TableDesc resultDesc = client.getTableDesc("nullcharsession"); + assertNullCharSessionVar(resultDesc); + } + + + public void assertNullCharSessionVar(TableDesc resultDesc) throws Exception { + TajoConf tajoConf = TpchTestBase.getInstance().getTestingCluster().getConfiguration(); - assertEquals(res.getTableDesc().getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T"); + assertEquals(resultDesc.getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T"); - Path path = new Path(res.getTableDesc().getPath()); + Path path = new Path(resultDesc.getPath()); FileSystem fs = path.getFileSystem(tajoConf); FileStatus[] files = fs.listStatus(path); http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 3b1b4e3..69c0e4b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -1706,13 +1706,25 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex createTableNode.setStorageType(CatalogProtos.StoreType.CSV); } - // Set default storage properties to be created. - KeyValueSet keyValueSet = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType()); + + + // Set default storage properties to table + KeyValueSet properties = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType()); + + // Priority to apply table properties + // 1. Explicit table properties specified in WITH clause + // 2. Session variables + + // Set session variables to properties + PlannerUtil.applySessionToTableProperties(context.queryContext, createTableNode.getStorageType(), properties); + // Set table properties specified in WITH clause if (expr.hasParams()) { - keyValueSet.putAll(expr.getParams()); + properties.putAll(expr.getParams()); } - createTableNode.setOptions(keyValueSet); + createTableNode.setOptions(properties); + + if (expr.hasPartition()) { if (expr.getPartitionMethod().getPartitionType().equals(PartitionType.COLUMN)) { http://git-wip-us.apache.org/repos/asf/tajo/blob/ab2efce8/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 6868b6c..c55c203 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -21,6 +21,8 @@ package org.apache.tajo.plan.util; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.Column; @@ -35,10 +37,15 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor; import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; import java.util.*; +import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.CSV; +import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.TEXTFILE; + public class PlannerUtil { public static boolean checkIfDDLPlan(LogicalNode node) { @@ -776,4 +783,14 @@ public class PlannerUtil { return explains.toString(); } + + public static void applySessionToTableProperties(OverridableConf sessionVars, + CatalogProtos.StoreType storeType, + KeyValueSet tableProperties) { + if (storeType == CSV || storeType == TEXTFILE) { + if (sessionVars.containsKey(SessionVars.NULL_CHAR)) { + tableProperties.set(StorageConstants.TEXT_NULL, sessionVars.get(SessionVars.NULL_CHAR)); + } + } + } }
