TAJO-928: Session variables should override query configs in TajoConf. Closes #98
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/ddfc3f33 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/ddfc3f33 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/ddfc3f33 Branch: refs/heads/master Commit: ddfc3f33039ee15ab0a2d3fe5890b9acb40aec3d Parents: 0603b49 Author: Hyunsik Choi <[email protected]> Authored: Mon Aug 11 12:52:52 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Mon Aug 11 12:52:52 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../java/org/apache/tajo/catalog/TableMeta.java | 14 +- .../apache/tajo/catalog/TestKeyValueSet.java | 83 ++++++- .../tajo/catalog/store/HCatalogStore.java | 28 +-- .../tajo/catalog/store/TestHCatalogStore.java | 12 +- .../org/apache/tajo/catalog/TestCatalog.java | 10 +- .../tajo/cli/DefaultTajoCliOutputFormatter.java | 16 +- .../java/org/apache/tajo/cli/HelpCommand.java | 100 ++++---- .../java/org/apache/tajo/cli/SetCommand.java | 78 +++++- .../main/java/org/apache/tajo/cli/TajoCli.java | 149 ++++++++---- .../apache/tajo/cli/TajoCliOutputFormatter.java | 5 +- .../org/apache/tajo/cli/TajoShellCommand.java | 10 +- .../main/java/org/apache/tajo/ConfigKey.java | 52 ++++ .../java/org/apache/tajo/OverridableConf.java | 235 +++++++++++++++++++ .../main/java/org/apache/tajo/SessionVars.java | 211 +++++++++++++++++ .../java/org/apache/tajo/conf/TajoConf.java | 199 ++++++++-------- .../main/java/org/apache/tajo/datum/Datum.java | 2 +- .../java/org/apache/tajo/datum/NullDatum.java | 1 + .../java/org/apache/tajo/util/KeyValueSet.java | 149 ++++++++++-- .../tajo/datum/TestArithmeticOperator.java | 4 +- .../tajo/engine/planner/LogicalOptimizer.java | 9 +- .../apache/tajo/engine/planner/LogicalPlan.java | 4 +- .../engine/planner/LogicalPlanPreprocessor.java | 18 +- .../engine/planner/LogicalPlanVerifier.java | 12 +- .../tajo/engine/planner/LogicalPlanner.java | 35 ++- .../engine/planner/PhysicalPlannerImpl.java | 44 +++- .../engine/planner/PreLogicalPlanVerifier.java | 21 +- .../engine/planner/global/GlobalPlanner.java | 18 +- .../planner/physical/ExternalSortExec.java | 4 +- .../engine/planner/physical/StoreTableExec.java | 6 +- .../apache/tajo/engine/query/QueryContext.java | 144 ++++++------ .../tajo/engine/query/QueryUnitRequestImpl.java | 3 +- .../org/apache/tajo/master/GlobalEngine.java | 94 ++++---- .../tajo/master/TajoMasterClientService.java | 31 ++- .../apache/tajo/master/querymaster/Query.java | 4 +- .../tajo/master/querymaster/QueryMaster.java | 11 +- .../querymaster/QueryMasterManagerService.java | 4 +- .../master/querymaster/QueryMasterTask.java | 4 +- .../tajo/master/querymaster/Repartitioner.java | 16 +- .../tajo/master/querymaster/SubQuery.java | 15 +- .../org/apache/tajo/master/session/Session.java | 50 ++-- .../session/SessionLivelinessMonitor.java | 2 +- .../main/java/org/apache/tajo/worker/Task.java | 4 +- .../apache/tajo/worker/TaskAttemptContext.java | 12 +- .../resources/webapps/worker/querydetail.jsp | 10 + .../apache/tajo/LocalTajoTestingUtility.java | 5 + .../java/org/apache/tajo/QueryTestCaseBase.java | 2 +- .../test/java/org/apache/tajo/TpchTestBase.java | 2 +- .../tajo/cli/TestDefaultCliOutputFormatter.java | 47 +++- .../java/org/apache/tajo/cli/TestTajoCli.java | 172 ++++++++------ .../org/apache/tajo/client/TestTajoClient.java | 8 +- .../apache/tajo/engine/eval/ExprTestBase.java | 9 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 16 +- .../engine/planner/TestLogicalOptimizer.java | 20 +- .../tajo/engine/planner/TestLogicalPlan.java | 3 +- .../tajo/engine/planner/TestLogicalPlanner.java | 135 ++++++++--- .../tajo/engine/planner/TestPlannerUtil.java | 5 +- .../planner/global/TestBroadcastJoinPlan.java | 62 ++--- .../planner/physical/TestBNLJoinExec.java | 9 +- .../planner/physical/TestBSTIndexExec.java | 4 +- .../planner/physical/TestExternalSortExec.java | 4 +- .../physical/TestFullOuterHashJoinExec.java | 24 +- .../physical/TestFullOuterMergeJoinExec.java | 29 +-- .../planner/physical/TestHashAntiJoinExec.java | 4 +- .../planner/physical/TestHashJoinExec.java | 19 +- .../planner/physical/TestHashSemiJoinExec.java | 4 +- .../physical/TestLeftOuterHashJoinExec.java | 28 +-- .../physical/TestLeftOuterNLJoinExec.java | 25 +- .../planner/physical/TestMergeJoinExec.java | 4 +- .../engine/planner/physical/TestNLJoinExec.java | 17 +- .../planner/physical/TestPhysicalPlanner.java | 100 ++++---- .../physical/TestProgressExternalSortExec.java | 4 +- .../physical/TestRightOuterHashJoinExec.java | 16 +- .../physical/TestRightOuterMergeJoinExec.java | 32 +-- .../engine/planner/physical/TestSortExec.java | 4 +- .../tajo/engine/query/TestGroupByQuery.java | 22 +- .../tajo/engine/query/TestJoinBroadcast.java | 8 +- .../apache/tajo/engine/query/TestJoinQuery.java | 90 +++---- .../tajo/engine/query/TestNullValues.java | 14 +- .../tajo/engine/query/TestSelectQuery.java | 10 +- .../apache/tajo/engine/query/TestSortQuery.java | 8 +- .../org/apache/tajo/jdbc/TestResultSet.java | 2 +- .../tajo/master/TestExecutionBlockCursor.java | 6 +- .../apache/tajo/master/TestGlobalPlanner.java | 4 +- .../querymaster/TestQueryUnitStatusUpdate.java | 2 +- .../tajo/worker/TestRangeRetrieverHandler.java | 8 +- .../TestTajoCli/testHelpSessionVars.result | 36 +++ ...estSelectResultWithNullTrueDeprecated.result | 9 + .../testStopWhenErrorDeprecated.result | 6 + .../java/org/apache/tajo/storage/Appender.java | 2 +- .../java/org/apache/tajo/storage/CSVFile.java | 11 +- .../org/apache/tajo/storage/StorageUtil.java | 18 +- .../org/apache/tajo/storage/avro/AvroUtil.java | 17 +- .../org/apache/tajo/storage/rcfile/RCFile.java | 10 +- .../sequencefile/SequenceFileAppender.java | 10 +- .../sequencefile/SequenceFileScanner.java | 5 +- .../apache/tajo/storage/v2/RCFileScanner.java | 2 +- 97 files changed, 2057 insertions(+), 992 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index deb41a0..182d07d 100644 --- a/CHANGES +++ b/CHANGES @@ -29,6 +29,9 @@ Release 0.9.0 - unreleased IMPROVEMENT + TAJO-928: Session variables should override query configs in TajoConf. + (hyunsik) + TAJO-989: Cleanup of child blocks after parent execution block is complete (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java index 6015fc9..2d95e6b 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableMeta.java @@ -78,7 +78,19 @@ public class TableMeta implements ProtoObject<CatalogProtos.TableProto>, GsonObj public void putOption(String key, String val) { maybeInitBuilder(); - options.put(key, val); + options.set(key, val); + } + + public boolean containsOption(String key) { + TableProtoOrBuilder p = viaProto ? proto : builder; + if (options != null) { + return this.options.containsKey(key); + } + if (!p.hasParams()) { + return false; + } + this.options = new KeyValueSet(p.getParams()); + return options.containsKey(key); } public String getOption(String key) { http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java index 39d5fbe..b317ba4 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestKeyValueSet.java @@ -24,23 +24,70 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; public class TestKeyValueSet { + @Test - public final void testPutAndGet() { + public final void testPutAndGetString() { KeyValueSet opts = new KeyValueSet(); - opts.put("name", "abc"); - opts.put("delimiter", ","); + opts.set("k1", "v1"); + opts.set("k2", "v2"); - assertEquals(",", opts.get("delimiter")); - assertEquals("abc", opts.get("name")); + assertEquals("v1", opts.get("k1")); + assertEquals("v2", opts.get("k2")); + assertEquals("default", opts.get("k3", "default")); } + @Test + public final void testPutAndGetBool() { + KeyValueSet opts = new KeyValueSet(); + opts.setBool("k1", true); + opts.setBool("k2", false); + + assertEquals(true, opts.getBool("k1")); + assertEquals(false, opts.getBool("k2")); + assertEquals(true, opts.getBool("k3", true)); + } + + @Test + public final void testPutAndGetInt() { + KeyValueSet opts = new KeyValueSet(); + opts.setInt("k1", 1980); + opts.setInt("k2", 401); + + assertEquals(1980, opts.getInt("k1")); + assertEquals(401, opts.getInt("k2")); + assertEquals(2020, opts.getInt("k3", 2020)); + } + + @Test + public final void testPutAndGetLong() { + KeyValueSet opts = new KeyValueSet(); + opts.setLong("k1", 1980); + opts.setLong("k2", 401); + + assertEquals(1980, opts.getLong("k1")); + assertEquals(401, opts.getLong("k2")); + assertEquals(2020, opts.getLong("k3", 2020l)); + } + + @Test + public final void testPutAndGetFloat() { + KeyValueSet opts = new KeyValueSet(); + opts.setFloat("k1", 1980.4f); + opts.setFloat("k2", 401.150f); + + assertTrue(1980.4f == opts.getFloat("k1")); + assertTrue(401.150f == opts.getFloat("k2")); + assertTrue(3.14f == opts.getFloat("k3", 3.14f)); + } + @Test public final void testGetProto() { KeyValueSet opts = new KeyValueSet(); - opts.put("name", "abc"); - opts.put("delimiter", ","); + opts.set("name", "abc"); + opts.set("delimiter", ","); PrimitiveProtos.KeyValueSetProto proto = opts.getProto(); KeyValueSet opts2 = new KeyValueSet(proto); @@ -49,16 +96,26 @@ public class TestKeyValueSet { } @Test - public final void testDelete() { + public final void testRemove() { KeyValueSet opts = new KeyValueSet(); - opts.put("name", "abc"); - opts.put("delimiter", ","); + opts.set("name", "abc"); + opts.set("delimiter", ","); assertEquals("abc", opts.get("name")); - assertEquals("abc", opts.delete("name")); - assertNull(opts.get("name")); + assertEquals("abc", opts.remove("name")); + try { + opts.get("name"); + assertTrue(false); + } catch (IllegalArgumentException iae) { + assertTrue(true); + } KeyValueSet opts2 = new KeyValueSet(opts.getProto()); - assertNull(opts2.get("name")); + try { + opts2.get("name"); + assertTrue(false); + } catch (IllegalArgumentException iae) { + assertTrue(true); + } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java index 6f48348..61bdccd 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java @@ -166,7 +166,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { stats = new TableStats(); options = new KeyValueSet(); options.putAll(table.getParameters()); - options.delete("EXTERNAL"); + options.remove("EXTERNAL"); Properties properties = table.getMetadata(); if (properties != null) { @@ -186,31 +186,31 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { } else { nullFormat = "\\N"; } - options.delete(serdeConstants.SERIALIZATION_NULL_FORMAT); + options.remove(serdeConstants.SERIALIZATION_NULL_FORMAT); // set file output format String fileOutputformat = properties.getProperty(hive_metastoreConstants.FILE_OUTPUT_FORMAT); storeType = CatalogUtil.getStoreType(HCatalogUtil.getStoreType(fileOutputformat)); if (storeType.equals(CatalogProtos.StoreType.CSV)) { - options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); - options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); + options.set(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); + options.set(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); } else if (storeType.equals(CatalogProtos.StoreType.RCFILE)) { - options.put(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); + options.set(StorageConstants.RCFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinaryColumnarSerDe.class.getName().equals(serde)) { - options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); } else if (ColumnarSerDe.class.getName().equals(serde)) { - options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } } else if (storeType.equals(CatalogProtos.StoreType.SEQUENCEFILE) ) { - options.put(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); - options.put(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); + options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StringEscapeUtils.escapeJava(fieldDelimiter)); + options.set(StorageConstants.SEQUENCEFILE_NULL, StringEscapeUtils.escapeJava(nullFormat)); String serde = properties.getProperty(serdeConstants.SERIALIZATION_LIB); if (LazyBinarySerDe.class.getName().equals(serde)) { - options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); } else if (LazySimpleSerDe.class.getName().equals(serde)) { - options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); } } @@ -481,7 +481,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe.class.getName()); } - if (tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL) != null) { + if (tableDesc.getMeta().getOptions().containsKey(StorageConstants.RCFILE_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.RCFILE_NULL))); } @@ -504,7 +504,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { StringEscapeUtils.unescapeJava(fieldDelimiter)); table.getParameters().remove(StorageConstants.CSVFILE_DELIMITER); - if (tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL) != null) { + if (tableDesc.getMeta().containsOption(StorageConstants.CSVFILE_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.CSVFILE_NULL))); table.getParameters().remove(StorageConstants.CSVFILE_NULL); @@ -534,7 +534,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore { sd.getSerdeInfo().setSerializationLib(org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe.class.getName()); } - if (tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL) != null) { + if (tableDesc.getMeta().containsOption(StorageConstants.SEQUENCEFILE_NULL)) { table.putToParameters(serdeConstants.SERIALIZATION_NULL_FORMAT, StringEscapeUtils.unescapeJava(tableDesc.getMeta().getOption(StorageConstants.SEQUENCEFILE_NULL))); table.getParameters().remove(StorageConstants.SEQUENCEFILE_NULL); http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java index a507b08..f225424 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java @@ -117,7 +117,7 @@ public class TestHCatalogStore { @Test public void testTableUsingRCFileWithBinarySerde() throws Exception { KeyValueSet options = new KeyValueSet(); - options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); @@ -146,7 +146,7 @@ public class TestHCatalogStore { @Test public void testTableUsingRCFileWithTextSerde() throws Exception { KeyValueSet options = new KeyValueSet(); - options.put(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); TableMeta meta = new TableMeta(CatalogProtos.StoreType.RCFILE, options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); @@ -174,8 +174,8 @@ public class TestHCatalogStore { @Test public void testTableWithNullValue() throws Exception { KeyValueSet options = new KeyValueSet(); - options.put(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002")); - options.put(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003")); + options.set(StorageConstants.CSVFILE_DELIMITER, StringEscapeUtils.escapeJava("\u0002")); + options.set(StorageConstants.CSVFILE_NULL, StringEscapeUtils.escapeJava("\u0003")); TableMeta meta = new TableMeta(CatalogProtos.StoreType.CSV, options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); @@ -315,7 +315,7 @@ public class TestHCatalogStore { @Test public void testTableUsingSequenceFileWithBinarySerde() throws Exception { KeyValueSet options = new KeyValueSet(); - options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); @@ -344,7 +344,7 @@ public class TestHCatalogStore { @Test public void testTableUsingSequenceFileWithTextSerde() throws Exception { KeyValueSet options = new KeyValueSet(); - options.put(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); TableMeta meta = new TableMeta(CatalogProtos.StoreType.SEQUENCEFILE, options); org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 4039ff2..654736d 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -565,7 +565,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); - opts.put("file.delimiter", ","); + opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); @@ -605,7 +605,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); - opts.put("file.delimiter", ","); + opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); Schema partSchema = new Schema(); @@ -643,7 +643,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); - opts.put("file.delimiter", ","); + opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); Schema partSchema = new Schema(); @@ -680,7 +680,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); - opts.put("file.delimiter", ","); + opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); Schema partSchema = new Schema(); @@ -717,7 +717,7 @@ public class TestCatalog { String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); - opts.put("file.delimiter", ","); + opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV, opts); Schema partSchema = new Schema(); http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java index 62736b5..543ac8c 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java @@ -20,11 +20,10 @@ package org.apache.tajo.cli; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.tajo.QueryId; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.client.QueryStatus; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.util.FileUtil; import java.io.InputStream; @@ -33,20 +32,17 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { - private TajoConf tajoConf; private int printPauseRecords; private boolean printPause; private boolean printErrorTrace; private String nullChar; @Override - public void init(TajoConf tajoConf) { - this.tajoConf = tajoConf; - - this.printPause = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_PAUSE); - this.printPauseRecords = tajoConf.getIntVar(TajoConf.ConfVars.CLI_PRINT_PAUSE_NUM_RECORDS); - this.printErrorTrace = tajoConf.getBoolVar(TajoConf.ConfVars.CLI_PRINT_ERROR_TRACE); - this.nullChar = tajoConf.getVar(ConfVars.CLI_NULL_CHAR); + public void init(TajoCli.TajoCliContext context) { + this.printPause = context.getBool(SessionVars.CLI_PAGING_ENABLED); + this.printPauseRecords = context.getInt(SessionVars.CLI_PAGE_ROWS); + this.printErrorTrace = context.getBool(SessionVars.CLI_DISPLAY_ERROR_TRACE); + this.nullChar = context.get(SessionVars.CLI_NULL_CHAR); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java index 8d908ec..52f13b0 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/HelpCommand.java @@ -40,53 +40,69 @@ public class HelpCommand extends TajoShellCommand { } @Override + public String [] getAliases() { + return new String [] {"\\help"}; + } + + @Override public void invoke(String[] cmd) throws Exception { if(targetDocVersion.equalsIgnoreCase("")) { targetDocVersion = getDocumentationVersion(); } - PrintWriter sout = context.getOutput(); - sout.println(); - - sout.println("General"); - sout.println(" \\copyright show Apache License 2.0"); - sout.println(" \\version show Tajo version"); - sout.println(" \\? show help"); - sout.println(" \\q quit tsql"); - sout.println(); - sout.println(); - - sout.println("Informational"); - sout.println(" \\l list databases"); - sout.println(" \\c show current database"); - sout.println(" \\c [DBNAME] connect to new database"); - sout.println(" \\d list tables"); - sout.println(" \\d [TBNAME] describe table"); - sout.println(" \\df list functions"); - sout.println(" \\df NAME describe function"); - sout.println(); - sout.println(); - - sout.println("Tool"); - sout.println(" \\! execute a linux shell command"); - sout.println(" \\dfs execute a dfs command"); - sout.println(" \\admin execute tajo admin command"); - sout.println(); - sout.println(); - - sout.println("Variables"); - sout.println(" \\set [[NAME] [VALUE] set session variable or list session variables"); - sout.println(" \\unset NAME unset session variable"); - sout.println(); - sout.println(); - - sout.println("Documentations"); - sout.println(" tsql guide http://tajo.apache.org/docs/"+ targetDocVersion +"/cli.html"); - sout.println(" Query language http://tajo.apache.org/docs/"+ targetDocVersion +"/sql_language.html"); - sout.println(" Functions http://tajo.apache.org/docs/"+ targetDocVersion +"/functions.html"); - sout.println(" Backup & restore http://tajo.apache.org/docs/"+ targetDocVersion +"/backup_and_restore.html"); - sout.println(" Configuration http://tajo.apache.org/docs/"+ targetDocVersion +"/configuration.html"); - sout.println(); + if (cmd.length == 1) { + PrintWriter sout = context.getOutput(); + sout.println(); + + sout.println("General"); + sout.println(" \\copyright show Apache License 2.0"); + sout.println(" \\version show Tajo version"); + sout.println(" \\? show help"); + sout.println(" \\? [COMMAND] show help of a given command"); + sout.println(" \\help alias of \\?"); + sout.println(" \\q quit tsql"); + sout.println(); + sout.println(); + + sout.println("Informational"); + sout.println(" \\l list databases"); + sout.println(" \\c show current database"); + sout.println(" \\c [DBNAME] connect to new database"); + sout.println(" \\d list tables"); + sout.println(" \\d [TBNAME] describe table"); + sout.println(" \\df list functions"); + sout.println(" \\df NAME describe function"); + sout.println(); + sout.println(); + + sout.println("Tool"); + sout.println(" \\! execute a linux shell command"); + sout.println(" \\dfs execute a dfs command"); + sout.println(" \\admin execute tajo admin command"); + sout.println(); + sout.println(); + + sout.println("Variables"); + sout.println(" \\set [[NAME] [VALUE] set session variable or list session variables"); + sout.println(" \\unset NAME unset session variable"); + sout.println(); + sout.println(); + + sout.println("Documentations"); + sout.println(" tsql guide http://tajo.apache.org/docs/"; + targetDocVersion + "/cli.html"); + sout.println(" Query language http://tajo.apache.org/docs/"; + targetDocVersion + "/sql_language.html"); + sout.println(" Functions http://tajo.apache.org/docs/"; + targetDocVersion + "/functions.html"); + sout.println(" Backup & restore http://tajo.apache.org/docs/"; + targetDocVersion + "/backup_and_restore.html"); + sout.println(" Configuration http://tajo.apache.org/docs/"; + targetDocVersion + "/configuration.html"); + sout.println(); + } else if (cmd.length == 2) { + String slashCommand = "\\" + cmd[1]; + if (context.getCommands().containsKey(slashCommand)) { + context.getCommands().get(slashCommand).printHelp(); + } else { + context.getOutput().println("Command not found: " + cmd[1]); + } + } } private String getDocumentationVersion() { http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java index c694507..c957a19 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/SetCommand.java @@ -18,11 +18,15 @@ package org.apache.tajo.cli; +import com.google.protobuf.ServiceException; +import org.apache.tajo.SessionVars; import org.apache.tajo.util.StringUtils; import java.util.HashMap; import java.util.Map; +import static org.apache.tajo.SessionVars.VariableMode; + public class SetCommand extends TajoShellCommand { public SetCommand(TajoCli.TajoCliContext context) { @@ -34,17 +38,46 @@ public class SetCommand extends TajoShellCommand { return "\\set"; } + private void showAllSessionVars() throws ServiceException { + for (Map.Entry<String, String> entry: client.getAllSessionVariables().entrySet()) { + context.getOutput().println(StringUtils.quote(entry.getKey()) + "=" + StringUtils.quote(entry.getValue())); + } + } + + private void updateSessionVariable(String key, String val) throws ServiceException { + Map<String, String> variables = new HashMap<String, String>(); + variables.put(key, val); + client.updateSessionVariables(variables); + } + + void set(String key, String val) throws ServiceException { + SessionVars sessionVar = null; + + if (SessionVars.exists(key)) { // if the variable is one of the session variables + sessionVar = SessionVars.get(key); + + // is it cli-side variable? + if (sessionVar.getMode() == VariableMode.CLI_SIDE_VAR) { + context.setCliSideVar(key, val); + } else { + updateSessionVariable(key, val); + } + + if (SessionVars.isDeprecated(key)) { + context.getOutput().println("Warning: deprecated to directly use config key in TajoConf.ConfVars. " + + "Please execute '\\help set'."); + } + } else { + updateSessionVariable(key, val); + } + } + @Override public void invoke(String[] cmd) throws Exception { if (cmd.length == 1) { - for (Map.Entry<String, String> entry: client.getAllSessionVariables().entrySet()) { - context.getOutput().println(StringUtils.quote(entry.getKey()) + "=" + StringUtils.quote(entry.getValue())); - } + showAllSessionVars(); } else if (cmd.length == 3) { - Map<String, String> variables = new HashMap<String, String>(); - variables.put(cmd[1], cmd[2]); - client.updateSessionVariables(variables); - context.setVariable(cmd[1], cmd[2]); + set(cmd[1], cmd[2]); } else { context.getOutput().println("usage: \\set [[NAME] VALUE]"); } @@ -59,4 +92,35 @@ public class SetCommand extends TajoShellCommand { public String getDescription() { return "set session variable or shows all session variables"; } + + @Override + public void printHelp() { + context.getOutput().println("\nAvailable Session Variables:\n"); + for (SessionVars var : SessionVars.values()) { + + if (var.getMode() == VariableMode.DEFAULT || + var.getMode() == VariableMode.CLI_SIDE_VAR || + var.getMode() == VariableMode.FROM_SHELL_ENV) { + + context.getOutput().println("\\set " + var.keyname() + " " + getDisplayType(var.getVarType()) + " - " + var + .getDescription()); + } + } + } + + public static String getDisplayType(Class<?> clazz) { + if (clazz == String.class) { + return "[text value]"; + } else if (clazz == Integer.class) { + return "[int value]"; + } else if (clazz == Long.class) { + return "[long value]"; + } else if (clazz == Float.class) { + return "[real value]"; + } else if (clazz == Boolean.class) { + return "[true or false]"; + } else { + return clazz.getSimpleName(); + } + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java index c20e44b..81427ab 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java @@ -18,11 +18,12 @@ package org.apache.tajo.cli; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.protobuf.ServiceException; import jline.console.ConsoleReader; import org.apache.commons.cli.*; -import org.apache.tajo.QueryId; -import org.apache.tajo.QueryIdFactory; +import org.apache.tajo.*; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; @@ -62,7 +63,7 @@ public class TajoCli { // Current States private String currentDatabase; - private TajoCliOutputFormatter outputFormatter; + private TajoCliOutputFormatter displayFormatter; private boolean wasError = false; @@ -99,7 +100,10 @@ public class TajoCli { options.addOption("help", "help", false, "help"); } - public class TajoCliContext { + public class TajoCliContext extends OverridableConf { + public TajoCliContext(TajoConf conf) { + super(conf, ConfigKey.ConfigType.SESSION); + } public TajoClient getTajoClient() { return client; @@ -121,18 +125,50 @@ public class TajoCli { return conf; } - public void setVariable(String key, String value) { - conf.set(key, value); - try { - initFormatter(); - } catch (Exception e) { - System.err.println(ERROR_PREFIX + e.getMessage()); + @VisibleForTesting + public String getCliSideVar(String key) { + if (SessionVars.exists(key)) { + ConfigKey configKey = SessionVars.get(key); + return get(configKey); + } else { + return get(key); + } + } + + public void setCliSideVar(String key, String value) { + Preconditions.checkNotNull(key); + Preconditions.checkNotNull(value); + + boolean shouldReloadFormatter = false; + + if (SessionVars.exists(key)) { + SessionVars configKey = SessionVars.get(key); + put(configKey, value); + shouldReloadFormatter = configKey.getMode() == SessionVars.VariableMode.CLI_SIDE_VAR; + } else { + set(key, value); + + // It is hard to recognize it is a client side variable. So, we always reload formatter. + shouldReloadFormatter = true; } + + if (shouldReloadFormatter) { + try { + initFormatter(); + } catch (Exception e) { + System.err.println(ERROR_PREFIX + e.getMessage()); + } + } + } + + public Map<String, TajoShellCommand> getCommands() { + return commands; } } public TajoCli(TajoConf c, String [] args, InputStream in, OutputStream out) throws Exception { this.conf = new TajoConf(c); + context = new TajoCliContext(conf); this.sin = in; this.reader = new ConsoleReader(sin, out); this.reader.setExpandEvents(false); @@ -161,13 +197,7 @@ public class TajoCli { } if (cmd.getOptionValues("conf") != null) { - for (String eachParam: cmd.getOptionValues("conf")) { - String[] tokens = eachParam.split("="); - if (tokens.length != 2) { - continue; - } - conf.set(tokens[0], tokens[1]); - } + processConfVarCommand(cmd.getOptionValues("conf")); } // if there is no "-h" option, @@ -196,19 +226,22 @@ public class TajoCli { client = new TajoClient(conf, baseDatabase); } - context = new TajoCliContext(); context.setCurrentDatabase(client.getCurrentDatabase()); initHistory(); initCommands(); + if (cmd.getOptionValues("conf") != null) { + processSessionVarCommand(cmd.getOptionValues("conf")); + } + if (cmd.hasOption("c")) { - outputFormatter.setScirptMode(); + displayFormatter.setScirptMode(); int exitCode = executeScript(cmd.getOptionValue("c")); sout.flush(); System.exit(exitCode); } if (cmd.hasOption("f")) { - outputFormatter.setScirptMode(); + displayFormatter.setScirptMode(); cmd.getOptionValues(""); File sqlFile = new File(cmd.getOptionValue("f")); if (sqlFile.exists()) { @@ -226,13 +259,38 @@ public class TajoCli { addShutdownHook(); } + private void processConfVarCommand(String[] confCommands) throws ServiceException { + for (String eachParam: confCommands) { + String[] tokens = eachParam.split("="); + if (tokens.length != 2) { + continue; + } + + if (!SessionVars.exists(tokens[0])) { + conf.set(tokens[0], tokens[1]); + } + } + } + + private void processSessionVarCommand(String[] confCommands) throws ServiceException { + for (String eachParam: confCommands) { + String[] tokens = eachParam.split("="); + if (tokens.length != 2) { + continue; + } + + if (SessionVars.exists(tokens[0])) { + ((SetCommand)commands.get("\\set")).set(tokens[0], tokens[1]); + } + } + } + private void initFormatter() throws Exception { - Class formatterClass = conf.getClass(ConfVars.CLI_OUTPUT_FORMATTER_CLASS.varname, - DefaultTajoCliOutputFormatter.class); - if (outputFormatter == null || !outputFormatter.getClass().equals(formatterClass)) { - outputFormatter = (TajoCliOutputFormatter)formatterClass.newInstance(); + Class formatterClass = context.getClass(SessionVars.CLI_FORMATTER_CLASS); + if (displayFormatter == null || !displayFormatter.getClass().equals(formatterClass)) { + displayFormatter = (TajoCliOutputFormatter)formatterClass.newInstance(); } - outputFormatter.init(conf); + displayFormatter.init(context); } public TajoCliContext getContext() { @@ -280,6 +338,9 @@ public class TajoCli { throw new RuntimeException(e.getMessage()); } commands.put(cmd.getCommand(), cmd); + for (String alias : cmd.getAliases()) { + commands.put(alias, cmd); + } } } @@ -332,7 +393,7 @@ public class TajoCli { exitCode = executeParsedResults(parsedResults); currentPrompt = updatePrompt(parser.getState()); - if (exitCode != 0 && context.getConf().getBoolVar(ConfVars.CLI_ERROR_STOP)) { + if (exitCode != 0 && context.getBool(SessionVars.ON_ERROR_STOP)) { return exitCode; } } @@ -372,18 +433,18 @@ public class TajoCli { try { invoked.invoke(arguments); } catch (IllegalArgumentException ige) { - outputFormatter.printErrorMessage(sout, ige); + displayFormatter.printErrorMessage(sout, ige); wasError = true; return -1; } catch (Exception e) { - outputFormatter.printErrorMessage(sout, e); + displayFormatter.printErrorMessage(sout, e); wasError = true; return -1; } finally { context.getOutput().flush(); } - if (wasError && context.getConf().getBoolVar(ConfVars.CLI_ERROR_STOP)) { + if (wasError && context.getBool(SessionVars.ON_ERROR_STOP)) { break; } } @@ -395,7 +456,7 @@ public class TajoCli { long startTime = System.currentTimeMillis(); ClientProtos.SubmitQueryResponse response = client.executeQueryWithJson(json); if (response == null) { - outputFormatter.printErrorMessage(sout, "response is null"); + displayFormatter.printErrorMessage(sout, "response is null"); wasError = true; } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { if (response.getIsForwarded()) { @@ -403,7 +464,7 @@ public class TajoCli { waitForQueryCompleted(queryId); } else { if (!response.hasTableDesc() && !response.hasResultSet()) { - outputFormatter.printMessage(sout, "OK"); + displayFormatter.printMessage(sout, "OK"); wasError = true; } else { localQueryCompleted(response, startTime); @@ -411,7 +472,7 @@ public class TajoCli { } } else { if (response.hasErrorMessage()) { - outputFormatter.printErrorMessage(sout, response.getErrorMessage()); + displayFormatter.printErrorMessage(sout, response.getErrorMessage()); wasError = true; } } @@ -421,7 +482,7 @@ public class TajoCli { long startTime = System.currentTimeMillis(); ClientProtos.SubmitQueryResponse response = client.executeQuery(statement); if (response == null) { - outputFormatter.printErrorMessage(sout, "response is null"); + displayFormatter.printErrorMessage(sout, "response is null"); wasError = true; } else if (response.getResultCode() == ClientProtos.ResultCode.OK) { if (response.getIsForwarded()) { @@ -429,14 +490,14 @@ public class TajoCli { waitForQueryCompleted(queryId); } else { if (!response.hasTableDesc() && !response.hasResultSet()) { - outputFormatter.printMessage(sout, "OK"); + displayFormatter.printMessage(sout, "OK"); } else { localQueryCompleted(response, startTime); } } } else { if (response.hasErrorMessage()) { - outputFormatter.printErrorMessage(sout, response.getErrorMessage()); + displayFormatter.printErrorMessage(sout, response.getErrorMessage()); wasError = true; } } @@ -454,13 +515,13 @@ public class TajoCli { // non-forwarded INSERT INTO query does not have any query id. // In this case, it just returns succeeded query information without printing the query results. if (response.getMaxRowNum() < 0 && queryId.equals(QueryIdFactory.NULL_QUERY_ID)) { - outputFormatter.printResult(sout, sin, desc, responseTime, res); + displayFormatter.printResult(sout, sin, desc, responseTime, res); } else { res = TajoClient.createResultSet(client, response); - outputFormatter.printResult(sout, sin, desc, responseTime, res); + displayFormatter.printResult(sout, sin, desc, responseTime, res); } } catch (Throwable t) { - outputFormatter.printErrorMessage(sout, t); + displayFormatter.printErrorMessage(sout, t); wasError = true; } finally { if (res != null) { @@ -495,7 +556,7 @@ public class TajoCli { } if (status.getState() == QueryState.QUERY_RUNNING || status.getState() == QueryState.QUERY_SUCCEEDED) { - outputFormatter.printProgress(sout, status); + displayFormatter.printProgress(sout, status); } if (status.getState() != QueryState.QUERY_RUNNING && @@ -509,10 +570,10 @@ public class TajoCli { } if (status.getState() == QueryState.QUERY_ERROR || status.getState() == QueryState.QUERY_FAILED) { - outputFormatter.printErrorMessage(sout, status); + displayFormatter.printErrorMessage(sout, status); wasError = true; } else if (status.getState() == QueryState.QUERY_KILLED) { - outputFormatter.printKilledMessage(sout, queryId); + displayFormatter.printKilledMessage(sout, queryId); wasError = true; } else { if (status.getState() == QueryState.QUERY_SUCCEEDED) { @@ -521,15 +582,15 @@ public class TajoCli { if (status.hasResult()) { res = TajoClient.createResultSet(client, queryId, response); TableDesc desc = new TableDesc(response.getTableDesc()); - outputFormatter.printResult(sout, sin, desc, responseTime, res); + displayFormatter.printResult(sout, sin, desc, responseTime, res); } else { TableDesc desc = new TableDesc(response.getTableDesc()); - outputFormatter.printResult(sout, sin, desc, responseTime, res); + displayFormatter.printResult(sout, sin, desc, responseTime, res); } } } } catch (Throwable t) { - outputFormatter.printErrorMessage(sout, t); + displayFormatter.printErrorMessage(sout, t); wasError = true; } finally { if (res != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java index 0e91669..0ad89f2 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCliOutputFormatter.java @@ -21,7 +21,6 @@ package org.apache.tajo.cli; import org.apache.tajo.QueryId; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; -import org.apache.tajo.conf.TajoConf; import java.io.InputStream; import java.io.PrintWriter; @@ -30,9 +29,9 @@ import java.sql.ResultSet; public interface TajoCliOutputFormatter { /** * Initialize formatter - * @param tajoConf + * @param context */ - public void init(TajoConf tajoConf); + public void init(TajoCli.TajoCliContext context); /** * print query result to console http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java index 1ea2893..39f5377 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoShellCommand.java @@ -23,16 +23,24 @@ import org.apache.tajo.conf.TajoConf; public abstract class TajoShellCommand { public abstract String getCommand(); + public String [] getAliases() { + return new String[] {}; + } public abstract void invoke(String [] command) throws Exception; public abstract String getUsage(); public abstract String getDescription(); + public void printHelp() { + context.getOutput().print(getCommand()); + context.getOutput().print(" - "); + context.getOutput().println(getDescription()); + } protected TajoCli.TajoCliContext context; protected TajoClient client; protected int maxColumn; public TajoShellCommand(TajoCli.TajoCliContext context) { - maxColumn = context.getTajoClient().getConf().getIntVar(TajoConf.ConfVars.CLI_MAX_COLUMN); + maxColumn = context.getTajoClient().getConf().getIntVar(TajoConf.ConfVars.$CLI_MAX_COLUMN); this.context = context; client = context.getTajoClient(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java new file mode 100644 index 0000000..b9d51ec --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +public interface ConfigKey { + + // Client can set or change variables of this mode. + public static final int DEFAULT_MODE = 0; + // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables. + public static final int FROM_SHELL_ENV_MODE = 1; + // only TajoMaster is able to set and change variables of this mode. + public static final int SERVER_SIDE_VAR_MODE = 2; + // This type variable will be used only in cli side. + public static final int CLI_SIDE_VAR_MODE = 3; + + public static enum ConfigType { + SYSTEM(""), + SESSION("$"), + QUERY("@"), + CLI("+"); + + private String prefix; + + ConfigType(String prefix) { + this.prefix = prefix; + } + + public String getPrefix() { + return prefix; + } + } + + public String keyname(); + + public ConfigType type(); +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java new file mode 100644 index 0000000..220bd43 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import com.google.common.base.Preconditions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.util.KeyValueSet; + +import static org.apache.tajo.ConfigKey.ConfigType; +import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; + +/** + * OverridableConf provides a consolidated config system. Tajo basically uses TajoConf, which is a extended class of + * Hadoop's Configuration system, However, TajoConf is only used for sharing static system configs, such as binding + * address of master and workers, system directories, other system parameters. + * + * For modifiable or instant configs, we use OverridableConf, which is a set of key-value pairs. + * OverridableConf provides more strong-typed way to set configs and its behavior is more clear than Configuration + * system. + * + * By default, OverridableConf recognizes following config types. + * + * <ul> + * <li>System Config - it comes from Hadoop's Configuration class. by tajo-site, catalog-site, + * catalog-default and TajoConf.</li> + * <li>Session variables - they are instantly configured by users. + * Each client session has it own set of session variables.</li> + * </ul> + * + * System configs and session variables can set the same config in the same time. System configs are usually used to set + * default configs, and session variables is user-specified configs. So, session variables can override system configs. + */ +public class OverridableConf extends KeyValueSet { + private static final Log LOG = LogFactory.getLog(OverridableConf.class); + private ConfigType [] configTypes; + private TajoConf conf; + + public OverridableConf(final TajoConf conf, ConfigType...configTypes) { + this.conf = conf; + this.configTypes = configTypes; + } + + public OverridableConf(final TajoConf conf, KeyValueSetProto proto, ConfigType...configTypes) { + super(proto); + this.conf = conf; + this.configTypes = configTypes; + } + + public void setConf(TajoConf conf) { + this.conf = conf; + } + + public TajoConf getConf() { + return conf; + } + + public void setBool(ConfigKey key, boolean val) { + setBool(key.keyname(), val); + } + + public boolean getBool(ConfigKey key, Boolean defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return getBool(key.keyname(), defaultVal); + } else { + switch (key.type()) { + case QUERY: + return getBool(key.keyname()); + case SESSION: + return getBool(key.keyname(), conf.getBoolVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getBoolVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public boolean getBool(ConfigKey key) { + return getBool(key, null); + } + + public int getInt(ConfigKey key, Integer defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return getInt(key.keyname(), defaultVal); + } else { + switch (key.type()) { + case SESSION: + return getInt(key.keyname(), conf.getIntVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getIntVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public int getInt(ConfigKey key) { + return getInt(key, null); + } + + public long getLong(ConfigKey key, Long defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return getLong(key.keyname(), defaultVal); + } else { + switch (key.type()) { + case SESSION: + return getLong(key.keyname(), conf.getLongVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getLongVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public long getLong(ConfigKey key) { + return getLong(key, null); + } + + public float getFloat(ConfigKey key, Float defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return getFloat(key.keyname(), defaultVal); + } else { + switch (key.type()) { + case SESSION: + return getFloat(key.keyname(), conf.getFloatVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getFloatVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public float getFloat(ConfigKey key) { + return getLong(key, null); + } + + public void put(ConfigKey key, String val) { + set(key.keyname(), val); + } + + private void assertRegisteredEnum(ConfigKey key) { + boolean registered = false; + + for (ConfigType c : configTypes) { + registered = key.type() == c; + } + + registered |= key.type() == ConfigType.SESSION || key.type() != ConfigType.SYSTEM; + + Preconditions.checkArgument(registered, key.keyname() + " (" + key.type() + ") is not allowed in " + + getClass().getSimpleName()); + } + + public String get(ConfigKey key, String defaultVal) { + assertRegisteredEnum(key); + + if (key.type() != ConfigType.SESSION && key.type() != ConfigType.SYSTEM) { + return get(key.keyname(), defaultVal); + } else { + switch (key.type()) { + case SESSION: + return get(key.keyname(), conf.getVar(((SessionVars) key).getConfVars())); + case SYSTEM: + return conf.getVar((TajoConf.ConfVars) key); + default: + throw new IllegalStateException("key does not belong to Session and System config sets"); + } + } + } + + public String get(ConfigKey key) { + return get(key, null); + } + + public Class<?> getClass(ConfigKey key) { + assertRegisteredEnum(key); + + String className = getTrimmed(key); + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + public String getTrimmed(ConfigKey key) { + String value = get(key); + + if (null == value) { + return null; + } else { + return value.trim(); + } + } + + public boolean containsKey(ConfigKey key) { + return containsKey(key.keyname()); + } + + public boolean equalKey(ConfigKey key, String another) { + if (containsKey(key)) { + return get(key).equals(another); + } else { + return false; + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/SessionVars.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java new file mode 100644 index 0000000..5503aaa --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import com.google.common.collect.Maps; + +import java.util.Map; + +import static org.apache.tajo.SessionVars.VariableMode.*; +import static org.apache.tajo.conf.TajoConf.ConfVars; + +public enum SessionVars implements ConfigKey { + + // Common Suffix Naming Rules: + // + // * LIMIT - We use the suffix 'LIMIT' if the variable is threshold. So, if some value is greater or less than + // the variable with suffix 'LIMIT', some action will be different from before. + // * SIZE - The suffix 'SIZE' means a data volume like bytes or mega bytes. + // It should be used for user's desired volume. + // * ENABLED - The suffix 'ENABLED' means a true or false value. If it is true, it will enable some feature. + // Otherwise, the feature will be turned off. + + + //------------------------------------------------------------------------------- + // Server Side Only Variables + //------------------------------------------------------------------------------- + SESSION_ID(ConfVars.$EMPTY, "session variable", SERVER_SIDE_VAR), + SESSION_LAST_ACCESS_TIME(ConfVars.$EMPTY, "last access time", SERVER_SIDE_VAR), + + USERNAME(ConfVars.USERNAME, "username", SERVER_SIDE_VAR), + CLIENT_HOST(ConfVars.$EMPTY, "client hostname", SERVER_SIDE_VAR), + + CURRENT_DATABASE(ConfVars.$EMPTY, "current database", SERVER_SIDE_VAR), + + //------------------------------------------------------------------------------- + // Client Side Variables + //------------------------------------------------------------------------------- + + // Client -------------------------------------------------------- + SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "session expiry time (secs)", DEFAULT), + + // Command line interface and its behavior -------------------------------- + CLI_COLUMNS(ConfVars.$CLI_MAX_COLUMN, "Sets the width for the wrapped format", CLI_SIDE_VAR), + CLI_FORMATTER_CLASS(ConfVars.$CLI_OUTPUT_FORMATTER_CLASS, "Sets the output format class to display results", + CLI_SIDE_VAR), + CLI_NULL_CHAR(ConfVars.$CLI_NULL_CHAR, "Sets the string to be printed in place of a null value.", CLI_SIDE_VAR), + + CLI_PAGE_ROWS(ConfVars.$CLI_PRINT_PAUSE_NUM_RECORDS, "Sets the number of rows for paging", CLI_SIDE_VAR), + CLI_PAGING_ENABLED(ConfVars.$CLI_PRINT_PAUSE, "Enable paging of result display", CLI_SIDE_VAR), + CLI_DISPLAY_ERROR_TRACE(ConfVars.$CLI_PRINT_ERROR_TRACE, "Enable display of error trace", CLI_SIDE_VAR), + + ON_ERROR_STOP(ConfVars.$CLI_ERROR_STOP, "tsql will exist if an error occurs.", CLI_SIDE_VAR), + + // Timezone & Date ---------------------------------------------------------- + TZ(ConfVars.$TIMEZONE, "Sets timezone", FROM_SHELL_ENV), + DATE_ORDER(ConfVars.$DATE_ORDER, "date order (default is YMD)", FROM_SHELL_ENV), + + // Locales and Character set ------------------------------------------------ + // TODO - they are reserved variables, and we should support them. + LANG(ConfVars.$EMPTY, "Language", FROM_SHELL_ENV), + LC_ALL(ConfVars.$EMPTY, "String sort order", FROM_SHELL_ENV), + LC_COLLATE(ConfVars.$EMPTY, "String sort order", FROM_SHELL_ENV), + LC_CTYPE(ConfVars.$EMPTY, "Character classification (What is a letter? Its upper-case equivalent?)", FROM_SHELL_ENV), + LC_MESSAGES(ConfVars.$EMPTY, "Language of messages", FROM_SHELL_ENV), + LC_MONETARY(ConfVars.$EMPTY, "Formatting of currency amounts", FROM_SHELL_ENV), + LC_NUMERIC(ConfVars.$EMPTY, "Formatting of numbers", FROM_SHELL_ENV), + LC_TIME(ConfVars.$EMPTY, "Formatting of dates and times", FROM_SHELL_ENV), + + + // Query and Optimization --------------------------------------------------- + + // for distributed query strategies + BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "limited size (bytes) of broadcast table", + DEFAULT), + + JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT), + SORT_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_SORT_TASK_VOLUME, "sort task input size (mb)", DEFAULT), + GROUPBY_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_GROUPBY_TASK_VOLUME, "group by task input size (mb)", DEFAULT), + + JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join (mb)", DEFAULT), + GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort (mb)", DEFAULT), + TABLE_PARTITION_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME, + "shuffle output size for partition table write (mb)", DEFAULT), + + // for physical Executors + EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT), + HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT), + INNER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD, + "limited size for hash inner join (mb)", DEFAULT), + OUTER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash outer join (mb)", + DEFAULT), + HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "limited size for hash groupby (mb)", + DEFAULT), + MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT), + + NULL_CHAR(ConfVars.$CSVFILE_NULL, "null char of text file output", DEFAULT), + + // Behavior Control --------------------------------------------------------- + ARITHABORT(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT, + "If true, a running query will be terminated when an overflow or divide-by-zero occurs.", DEFAULT), + + //------------------------------------------------------------------------------- + // Only for Unit Testing + //------------------------------------------------------------------------------- + DEBUG_ENABLED(ConfVars.$DEBUG_ENABLED, "(debug only) debug mode enabled", DEFAULT), + TEST_BROADCAST_JOIN_ENABLED(ConfVars.$TEST_BROADCAST_JOIN_ENABLED, "(test only) broadcast enabled", TEST_VAR), + TEST_JOIN_OPT_ENABLED(ConfVars.$TEST_JOIN_OPT_ENABLED, "(test only) join optimization enabled", TEST_VAR), + TEST_FILTER_PUSHDOWN_ENABLED(ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED, "filter push down enabled", TEST_VAR), + TEST_MIN_TASK_NUM(ConfVars.$TEST_MIN_TASK_NUM, "(test only) min task num", TEST_VAR), + ; + + public static Map<String, SessionVars> SESSION_VARS = Maps.newHashMap(); + public static Map<String, SessionVars> DEPRECATED_SESSION_VARS = Maps.newHashMap(); + + static { + for (SessionVars var : SessionVars.values()) { + SESSION_VARS.put(var.keyname(), var); + DEPRECATED_SESSION_VARS.put(var.getConfVars().keyname(), var); + } + } + + private final ConfVars key; + private final String description; + private final VariableMode mode; + + public static enum VariableMode { + DEFAULT, // Client can set or change variables of this mode.. + FROM_SHELL_ENV, // This is similar to DEFAULT mode. In addition, it tries to get values from shell env. variables. + SERVER_SIDE_VAR, // only TajoMaster is able to set and change variables of this mode. + CLI_SIDE_VAR, // This type variable is used in CLI. + TEST_VAR // Only used for unit tests + } + + SessionVars(ConfVars key, String description, VariableMode mode) { + this.key = key; + this.description = description; + this.mode = mode; + } + + public String keyname() { + return name(); + } + + public ConfigType type() { + return ConfigType.SESSION; + } + + public ConfVars getConfVars() { + return key; + } + + public Class<?> getVarType() { + return key.valClass; + } + + public String getDescription() { + return description; + } + + public VariableMode getMode() { + return mode; + } + + public static boolean exists(String keyname) { + return SESSION_VARS.containsKey(keyname) || DEPRECATED_SESSION_VARS.containsKey(keyname); + } + + public static boolean isDeprecated(String keyname) { + return DEPRECATED_SESSION_VARS.containsKey(keyname); + } + + public static boolean isPublic(SessionVars var) { + return var.getMode() != SERVER_SIDE_VAR; + } + + public static SessionVars get(String keyname) { + if (SESSION_VARS.containsKey(keyname)) { + return SESSION_VARS.get(keyname); + } else if (DEPRECATED_SESSION_VARS.containsKey(keyname)) { + return DEPRECATED_SESSION_VARS.get(keyname); + } else { + return null; + } + } + + /** + * rename deprecated name to current name if the name is deprecated. + * + * @param keyname session variable name + * @return The current session variable name + */ + public static String handleDeprecatedName(String keyname) { + return SessionVars.exists(keyname) ? SessionVars.get(keyname).keyname() : keyname; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index b75530b..2504c23 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.ConfigKey; import org.apache.tajo.TajoConstants; import org.apache.tajo.util.NetUtils; import org.apache.tajo.util.TUtil; @@ -81,7 +82,7 @@ public class TajoConf extends Configuration { try { if (CURRENT_TIMEZONE == null) { TajoConf tajoConf = new TajoConf(); - CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.TAJO_TIMEZONE)); + CURRENT_TIMEZONE = TimeZone.getTimeZone(tajoConf.getVar(ConfVars.$TIMEZONE)); } return CURRENT_TIMEZONE; } finally { @@ -105,7 +106,7 @@ public class TajoConf extends Configuration { try { if (DATE_ORDER < 0) { TajoConf tajoConf = new TajoConf(); - String dateOrder = tajoConf.getVar(ConfVars.TAJO_DATE_ORDER); + String dateOrder = tajoConf.getVar(ConfVars.$DATE_ORDER); if ("YMD".equals(dateOrder)) { DATE_ORDER = DateTimeConstants.DATEORDER_YMD; } else if ("DMY".equals(dateOrder)) { @@ -133,11 +134,13 @@ public class TajoConf extends Configuration { } } - public static enum ConfVars { + public static enum ConfVars implements ConfigKey { - ////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////////// // Tajo System Configuration - ////////////////////////////////// + // + // They are all static configs which are not changed or not overwritten at all. + /////////////////////////////////////////////////////////////////////////////////////// // a username for a running Tajo cluster ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/"), @@ -196,23 +199,19 @@ public class TajoConf extends Configuration { // Catalog CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005"), - ////////////////////////////////// - // for Yarn Resource Manager - ////////////////////////////////// + + // for Yarn Resource Manager ---------------------------------------------- + /** how many launching TaskRunners in parallel */ YARN_RM_QUERY_MASTER_MEMORY_MB("tajo.querymaster.memory-mb", 512), YARN_RM_QUERY_MASTER_DISKS("tajo.yarn-rm.querymaster.disks", 1), YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM("tajo.yarn-rm.parallel-task-runner-launcher-num", 16), YARN_RM_WORKER_NUMBER_PER_NODE("tajo.yarn-rm.max-worker-num-per-node", 8), - ////////////////////////////////// // Query Configuration - ////////////////////////////////// QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60), - ////////////////////////////////// - // Shuffle Configuration - ////////////////////////////////// + // Shuffle Configuration -------------------------------------------------- PULLSERVER_PORT("tajo.pullserver.port", 0), SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false), SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"), @@ -221,61 +220,28 @@ public class TajoConf extends Configuration { SHUFFLE_FETCHER_READ_TIMEOUT("tajo.shuffle.fetcher.read.timeout-sec", 120), SHUFFLE_FETCHER_READ_RETRY_MAX_NUM("tajo.shuffle.fetcher.read.retry.max-num", 20), - ////////////////////////////////// - // Storage Configuration - ////////////////////////////////// + + // Storage Configuration -------------------------------------------------- ROWFILE_SYNC_INTERVAL("rowfile.sync.interval", 100), MINIMUM_SPLIT_SIZE("tajo.min.split.size", (long) 1), // for RCFile HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true), - // for Storage Manager v2 + // for Storage Manager ---------------------------------------------------- STORAGE_MANAGER_VERSION_2("tajo.storage-manager.v2", false), STORAGE_MANAGER_DISK_SCHEDULER_MAX_READ_BYTES_PER_SLOT("tajo.storage-manager.max-read-bytes", 8 * 1024 * 1024), STORAGE_MANAGER_DISK_SCHEDULER_REPORT_INTERVAL("tajo.storage-manager.disk-scheduler.report-interval", 60 * 1000), STORAGE_MANAGER_CONCURRENCY_PER_DISK("tajo.storage-manager.disk-scheduler.per-disk-concurrency", 2), - ////////////////////////////////////////// - // Distributed Query Execution Parameters - ////////////////////////////////////////// - DIST_QUERY_BROADCAST_JOIN_AUTO("tajo.dist-query.join.auto-broadcast", true), - DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576), - - DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128), - DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128), - DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128), - - DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128), - DIST_QUERY_SORT_PARTITION_VOLUME("tajo.dist-query.sort.partition-volume-mb", 256), - DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256), - - DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256), - - ////////////////////////////////// - // Physical Executors - ////////////////////////////////// - EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1), - EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L), - EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8), - - EXECUTOR_INNER_JOIN_INMEMORY_HASH_TABLE_SIZE("tajo.executor.join.inner.in-memory-table-num", (long)1000000), - EXECUTOR_INNER_JOIN_INMEMORY_HASH_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-bytes", - (long)256 * 1048576), - EXECUTOR_OUTER_JOIN_INMEMORY_HASH_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-bytes", - (long)256 * 1048576), - EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes", - (long)256 * 1048576), - ////////////////////////////////// - // RPC - ////////////////////////////////// + // RPC -------------------------------------------------------------------- RPC_POOL_MAX_IDLE("tajo.rpc.pool.idle.max", 10), - //Internal RPC Client + // Internal RPC Client INTERNAL_RPC_CLIENT_WORKER_THREAD_NUM("tajo.internal.rpc.client.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2), - //Internal RPC Server + // Internal RPC Server MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.master.rpc.server.worker-thread-num", Runtime.getRuntime().availableProcessors() * 2), QUERY_MASTER_RPC_SERVER_WORKER_THREAD_NUM("tajo.querymaster.rpc.server.worker-thread-num", @@ -296,58 +262,92 @@ public class TajoConf extends Configuration { WORKER_SERVICE_RPC_SERVER_WORKER_THREAD_NUM("tajo.worker.service.rpc.server.worker-thread-num", Runtime.getRuntime().availableProcessors() * 1), - ////////////////////////////////// - // The Below is reserved - ////////////////////////////////// - - // GeoIP - GEOIP_DATA("tajo.function.geoip-database-location", ""), - - ////////////////////////////////// - // Task Configuration + // Task Configuration ----------------------------------------------------- TASK_DEFAULT_MEMORY("tajo.task.memory-slot-mb.default", 512), TASK_DEFAULT_DISK("tajo.task.disk-slot.default", 0.5f), TASK_DEFAULT_SIZE("tajo.task.size-mb", 128), - ////////////////////////////////// - ////////////////////////////////// - // User Session Configuration - ////////////////////////////////// - CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour. + // Query and Optimization ------------------------------------------------- + EXECUTOR_EXTERNAL_SORT_THREAD_NUM("tajo.executor.external-sort.thread-num", 1), + EXECUTOR_EXTERNAL_SORT_FANOUT("tajo.executor.external-sort.fanout-num", 8), + + EXECUTOR_INNER_JOIN_INMEMORY_HASH_TABLE_SIZE("tajo.executor.join.inner.in-memory-table-num", (long)1000000), - // Metrics + // Metrics ---------------------------------------------------------------- METRICS_PROPERTY_FILENAME("tajo.metrics.property.file", "tajo-metrics.properties"), - //CLI - CLI_MAX_COLUMN("tajo.cli.max_columns", 120), - CLI_PRINT_PAUSE_NUM_RECORDS("tajo.cli.print.pause.num.records", 100), - CLI_PRINT_PAUSE("tajo.cli.print.pause", true), - CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true), - CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"), - CLI_NULL_CHAR("tajo.cli.nullchar", ""), - CLI_ERROR_STOP("tajo.cli.error.stop", false), + // Misc ------------------------------------------------------------------- - //TIME & DATE - TAJO_TIMEZONE("tajo.timezone", System.getProperty("user.timezone")), - TAJO_DATE_ORDER("tajo.date.order", "YMD"), + // Geo IP + GEOIP_DATA("tajo.function.geoip-database-location", ""), - //PLANNER - PLANNER_USE_FILTER_PUSHDOWN("tajo.planner.use.filter.pushdown", true), + ///////////////////////////////////////////////////////////////////////////////// + // User Session Configuration + // + // All session variables begin with dollor($) sign. They are default configs + // for session variables. Do not directly use the following configs. Instead, + // please use QueryContext in order to access session variables. + // + // Also, users can change the default values of session variables in tajo-site.xml. + ///////////////////////////////////////////////////////////////////////////////// + + + $EMPTY("tajo._", ""), + + // Query and Optimization --------------------------------------------------- + + // for distributed query strategies + $DIST_QUERY_BROADCAST_JOIN_THRESHOLD("tajo.dist-query.join.broadcast.threshold-bytes", (long)5 * 1048576), + + $DIST_QUERY_JOIN_TASK_VOLUME("tajo.dist-query.join.task-volume-mb", 128), + $DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128), + $DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128), + + $DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128), + $DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256), + $DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256), + + // for physical Executors + $EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE("tajo.executor.external-sort.buffer-mb", 200L), + $EXECUTOR_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.common.in-memory-hash-threshold-bytes", + (long)256 * 1048576), + $EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.inner.in-memory-hash-threshold-bytes", + (long)256 * 1048576), + $EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD("tajo.executor.join.outer.in-memory-hash-threshold-bytes", + (long)256 * 1048576), + $EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD("tajo.executor.groupby.in-memory-hash-threshold-bytes", + (long)256 * 1048576), + $MAX_OUTPUT_FILE_SIZE("tajo.query.max-outfile-size-mb", 0), // zero means infinite - // FILE FORMAT - CSVFILE_NULL("tajo.csvfile.null", "\\\\N"), - //OPTIMIZER - OPTIMIZER_JOIN_ENABLE("tajo.optimizer.join.enable", true), + // Client ----------------------------------------------------------------- + $CLIENT_SESSION_EXPIRY_TIME("tajo.client.session.expiry-time-sec", 3600), // default time is one hour. - // DEBUG OPTION - TAJO_DEBUG("tajo.debug", false), + // Command line interface and its behavior -------------------------------- + $CLI_MAX_COLUMN("tajo.cli.max_columns", 120), + $CLI_NULL_CHAR("tajo.cli.nullchar", ""), + $CLI_PRINT_PAUSE_NUM_RECORDS("tajo.cli.print.pause.num.records", 100), + $CLI_PRINT_PAUSE("tajo.cli.print.pause", true), + $CLI_PRINT_ERROR_TRACE("tajo.cli.print.error.trace", true), + $CLI_OUTPUT_FORMATTER_CLASS("tajo.cli.output.formatter", "org.apache.tajo.cli.DefaultTajoCliOutputFormatter"), + $CLI_ERROR_STOP("tajo.cli.error.stop", false), - // ONLY FOR TESTCASE - TESTCASE_MIN_TASK_NUM("tajo.testcase.min.task.num", -1), + // Timezone & Date ---------------------------------------------------------- + $TIMEZONE("tajo.timezone", System.getProperty("user.timezone")), + $DATE_ORDER("tajo.date.order", "YMD"), - // behavior control - BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false); + // FILE FORMAT + $CSVFILE_NULL("tajo.csvfile.null", "\\\\N"), + + // Only for Debug and Testing + $DEBUG_ENABLED("tajo.debug.enabled", false), + $TEST_BROADCAST_JOIN_ENABLED("tajo.dist-query.join.auto-broadcast", true), + $TEST_JOIN_OPT_ENABLED("tajo.test.plan.join-optimization.enabled", true), + $TEST_FILTER_PUSHDOWN_ENABLED("tajo.test.plan.filter-pushdown.enabled", true), + $TEST_MIN_TASK_NUM("tajo.test.min-task-num", -1), + + // Behavior Control --------------------------------------------------------- + $BEHAVIOR_ARITHMETIC_ABORT("tajo.behavior.arithmetic-abort", false); ; public final String varname; @@ -429,6 +429,16 @@ public class TajoConf extends Configuration { String typeString() { return name().toUpperCase();} abstract void checkType(String value) throws Exception; } + + @Override + public String keyname() { + return varname; + } + + @Override + public ConfigType type() { + return ConfigType.SYSTEM; + } } public static int getIntVar(Configuration conf, ConfVars var) { @@ -450,8 +460,12 @@ public class TajoConf extends Configuration { } public static long getLongVar(Configuration conf, ConfVars var) { - assert (var.valClass == Long.class); - return conf.getLong(var.varname, var.defaultLongVal); + assert (var.valClass == Long.class || var.valClass == Integer.class); + if (var.valClass == Integer.class) { + return conf.getInt(var.varname, var.defaultIntVal); + } else { + return conf.getLong(var.varname, var.defaultLongVal); + } } public static long getLongVar(Configuration conf, ConfVars var, long defaultVal) { @@ -516,7 +530,6 @@ public class TajoConf extends Configuration { } public static String getVar(Configuration conf, ConfVars var) { - assert (var.valClass == String.class); return conf.get(var.varname, var.defaultVal); } http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java index 19acafc..874004b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/Datum.java @@ -114,7 +114,7 @@ public abstract class Datum implements Comparable<Datum>, GsonObject { } protected static void initAbortWhenDivideByZero(TajoConf tajoConf) { - abortWhenDivideByZero = tajoConf.getBoolVar(ConfVars.BEHAVIOR_ARITHMETIC_ABORT); + abortWhenDivideByZero = tajoConf.getBoolVar(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT); } public abstract int size(); http://git-wip-us.apache.org/repos/asf/tajo/blob/ddfc3f33/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java index a4f79d7..532e7cd 100644 --- a/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java +++ b/tajo-common/src/main/java/org/apache/tajo/datum/NullDatum.java @@ -25,6 +25,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type; public class NullDatum extends Datum { private static NullDatum instance; + public static final String DEFAULT_TEXT = ""; private static final byte [] EMPTY_BYTES = new byte[0]; private static final DataType NULL_DATA_TYPE;
