Repository: tajo Updated Branches: refs/heads/master 95cf4b943 -> facd1ddcc
http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java index 54c8722..bec8cd3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestConditionalExpressions.java @@ -212,25 +212,25 @@ public class TestConditionalExpressions extends ExprTestBase { @Test public void testCoalesceTimestamp() throws Exception { testSimpleEval("select coalesce(null, timestamp '2014-01-01 00:00:00');", - new String[]{"2014-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-01 00:00:00"}); testSimpleEval("select coalesce(null, null, timestamp '2014-01-01 00:00:00');", - new String[]{"2014-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-01 00:00:00"}); testSimpleEval("select coalesce(timestamp '2014-01-01 00:00:00', null, timestamp '2014-01-02 00:00:00');", - new String[]{"2014-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-01 00:00:00"}); testSimpleEval("select coalesce(null, timestamp '2014-01-01 00:00:00', timestamp '2014-02-01 00:00:00');", - new String[]{"2014-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-01 00:00:00"}); } @Test public void testCoalesceTime() throws Exception { testSimpleEval("select coalesce(null, time '12:00:00');", - new String[]{"12:00:00" + getUserTimeZoneDisplay()}); + new String[]{"12:00:00"}); testSimpleEval("select coalesce(null, null, time '12:00:00');", - new String[]{"12:00:00" + getUserTimeZoneDisplay()}); + new String[]{"12:00:00"}); testSimpleEval("select coalesce(time '12:00:00', null, time '13:00:00');", - new String[]{"12:00:00" + getUserTimeZoneDisplay()}); + new String[]{"12:00:00"}); testSimpleEval("select coalesce(null, time '12:00:00', time '13:00:00');", - new String[]{"12:00:00" + getUserTimeZoneDisplay()}); + new String[]{"12:00:00"}); } @Test http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java index 7cca13d..cb7856b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestDateTimeFunctions.java @@ -19,11 +19,13 @@ package org.apache.tajo.engine.function; +import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Schema; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TimestampDatum; import org.apache.tajo.engine.eval.ExprTestBase; +import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.util.datetime.TimeMeta; import org.junit.Test; @@ -34,6 +36,7 @@ import java.util.Date; import java.util.TimeZone; import static org.apache.tajo.common.TajoDataTypes.Type.*; +import static org.junit.Assert.assertEquals; public class TestDateTimeFunctions extends ExprTestBase { @Test @@ -43,58 +46,58 @@ public class TestDateTimeFunctions extends ExprTestBase { // (expectedTimestamp / 1000) means the translation from millis seconds to unix timestamp String q1 = String.format("select to_timestamp(%d);", (expectedTimestamp / 1000)); - testSimpleEval(q1, new String[]{expected.toString(TajoConf.getCurrentTimeZone(), true)}); + testSimpleEval(q1, new String[]{expected.toString()}); testSimpleEval("select to_timestamp('1997-12-30 11:40:50.345', 'YYYY-MM-DD HH24:MI:SS.MS');", - new String[]{"1997-12-30 11:40:50.345" + getUserTimeZoneDisplay()}); + new String[]{"1997-12-30 11:40:50.345"}); testSimpleEval("select to_timestamp('1997-12-30 11:40:50.345 PM', 'YYYY-MM-DD HH24:MI:SS.MS PM');", - new String[]{"1997-12-30 23:40:50.345" + getUserTimeZoneDisplay()}); + new String[]{"1997-12-30 23:40:50.345"}); testSimpleEval("select to_timestamp('0097/Feb/16 --> 08:14:30', 'YYYY/Mon/DD --> HH:MI:SS');", - new String[]{"0097-02-16 08:14:30" + getUserTimeZoneDisplay()}); + new String[]{"0097-02-16 08:14:30"}); testSimpleEval("select to_timestamp('97/2/16 8:14:30', 'FMYYYY/FMMM/FMDD FMHH:FMMI:FMSS');", - new String[]{"0097-02-16 08:14:30" + getUserTimeZoneDisplay()}); + new String[]{"0097-02-16 08:14:30"}); testSimpleEval("select to_timestamp('1985 September 12', 'YYYY FMMonth DD');", - new String[]{"1985-09-12 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1985-09-12 00:00:00"}); testSimpleEval("select to_timestamp('1,582nd VIII 21', 'Y,YYYth FMRM DD');", - new String[]{"1582-08-21 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1582-08-21 00:00:00"}); testSimpleEval("select to_timestamp('05121445482000', 'MMDDHH24MISSYYYY');", - new String[]{"2000-05-12 14:45:48" + getUserTimeZoneDisplay()}); + new String[]{"2000-05-12 14:45:48"}); testSimpleEval("select to_timestamp('2000January09Sunday', 'YYYYFMMonthDDFMDay');", - new String[]{"2000-01-09 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2000-01-09 00:00:00"}); testSimpleEval("select to_timestamp('97/Feb/16', 'YY/Mon/DD');", - new String[]{"1997-02-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1997-02-16 00:00:00"}); testSimpleEval("select to_timestamp('19971116', 'YYYYMMDD');", - new String[]{"1997-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1997-11-16 00:00:00"}); testSimpleEval("select to_timestamp('20000-1116', 'YYYY-MMDD');", - new String[]{"20000-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"20000-11-16 00:00:00"}); testSimpleEval("select to_timestamp('9-1116', 'Y-MMDD');", - new String[]{"2009-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2009-11-16 00:00:00"}); testSimpleEval("select to_timestamp('95-1116', 'YY-MMDD');", - new String[]{"1995-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1995-11-16 00:00:00"}); testSimpleEval("select to_timestamp('995-1116', 'YYY-MMDD');", - new String[]{"1995-11-16 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"1995-11-16 00:00:00"}); testSimpleEval("select to_timestamp('2005426', 'YYYYWWD');", - new String[]{"2005-10-15 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-10-15 00:00:00"}); testSimpleEval("select to_timestamp('2005300', 'YYYYDDD');", - new String[]{"2005-10-27 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-10-27 00:00:00"}); testSimpleEval("select to_timestamp('2005527', 'IYYYIWID');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('005527', 'IYYIWID');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('05527', 'IYIWID');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('5527', 'IIWID');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('2005364', 'IYYYIDDD');", - new String[]{"2006-01-01 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2006-01-01 00:00:00"}); testSimpleEval("select to_timestamp('20050302', 'YYYYMMDD');", - new String[]{"2005-03-02 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-03-02 00:00:00"}); testSimpleEval("select to_timestamp('2005 03 02', 'YYYYMMDD');", - new String[]{"2005-03-02 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-03-02 00:00:00"}); testSimpleEval("select to_timestamp(' 2005 03 02', 'YYYYMMDD');", - new String[]{"2005-03-02 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-03-02 00:00:00"}); testSimpleEval("select to_timestamp(' 20050302', 'YYYYMMDD');", - new String[]{"2005-03-02 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2005-03-02 00:00:00"}); } @Test @@ -115,10 +118,21 @@ public class TestDateTimeFunctions extends ExprTestBase { @Test public void testExtract() throws IOException { + TimeZone GMT = TimeZone.getTimeZone("GMT"); + TimeZone PST = TimeZone.getTimeZone("PST"); + Schema schema2 = new Schema(); schema2.addColumn("col1", TIMESTAMP); testEval(schema2, "table1", - "1970-01-17 10:09:37" + getUserTimeZoneDisplay(), + "1970-01-17 10:09:37", + "select extract(year from col1), extract(month from col1), extract(day from col1) from table1;", + new String[]{"1970.0", "1.0", "17.0"}); + testEval(schema2, "table1", + "1970-01-17 10:09:37" + getUserTimeZoneDisplay(GMT), + "select extract(year from col1), extract(month from col1), extract(day from col1) from table1;", + new String[]{"1970.0", "1.0", "17.0"}); + testEval(schema2, "table1", + "1970-01-17 10:09:37" + getUserTimeZoneDisplay(PST), "select extract(year from col1), extract(month from col1), extract(day from col1) from table1;", new String[]{"1970.0", "1.0", "17.0"}); @@ -126,9 +140,17 @@ public class TestDateTimeFunctions extends ExprTestBase { Schema schema3 = new Schema(); schema3.addColumn("col1", TIME); testEval(schema3, "table1", - "10:09:37.5" + getUserTimeZoneDisplay(), + "10:09:37.5", "select extract(hour from col1), extract(minute from col1), extract(second from col1) from table1;", new String[]{"10.0", "9.0", "37.5"}); + testEval(schema3, "table1", + "10:09:37.5" + getUserTimeZoneDisplay(GMT), + "select extract(hour from col1), extract(minute from col1), extract(second from col1) from table1;", + new String[]{"10.0", "9.0", "37.5"}); + testEval(schema3, "table1", + "10:09:37.5" + getUserTimeZoneDisplay(PST), + "select extract(hour from col1), extract(minute from col1), extract(second from col1) from table1;", + new String[]{"18.0", "9.0", "37.5"}); Schema schema4 = new Schema(); schema4.addColumn("col1", DATE); @@ -212,18 +234,36 @@ public class TestDateTimeFunctions extends ExprTestBase { @Test public void testDatePart() throws IOException { + TimeZone GMT = TimeZone.getTimeZone("GMT"); + TimeZone PST = TimeZone.getTimeZone("PST"); + Schema schema2 = new Schema(); schema2.addColumn("col1", TIMESTAMP); + + testEval(schema2, "table1", + "1970-01-17 22:09:37", + "select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;", + new String[]{"1970.0", "1.0", "17.0"}); testEval(schema2, "table1", - "1970-01-17 22:09:37" + getUserTimeZoneDisplay(), + "1970-01-17 22:09:37" + getUserTimeZoneDisplay(GMT), "select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;", new String[]{"1970.0", "1.0", "17.0"}); + testEval(schema2, "table1", + "1970-01-17 22:09:37" + getUserTimeZoneDisplay(PST), + "select date_part('year', col1), date_part('month', col1), date_part('day', col1) from table1;", + new String[]{"1970.0", "1.0", "18.0"}); Schema schema3 = new Schema(); schema3.addColumn("col1", TIME); - testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(), + testEval(schema3, "table1", "10:09:37.5", "select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;", new String[]{"10.0", "9.0", "37.5"}); + testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(GMT), + "select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;", + new String[]{"10.0", "9.0", "37.5"}); + testEval(schema3, "table1", "10:09:37.5" + getUserTimeZoneDisplay(PST), + "select date_part('hour', col1), date_part('minute', col1), date_part('second', col1) from table1;", + new String[]{"18.0", "9.0", "37.5"}); Schema schema4 = new Schema(); schema4.addColumn("col1", DATE); @@ -318,7 +358,7 @@ public class TestDateTimeFunctions extends ExprTestBase { public void testToDate() throws IOException { testSimpleEval("select to_date('2014-01-04', 'YYYY-MM-DD')", new String[]{"2014-01-04"}); testSimpleEval("select to_date('2014-01-04', 'YYYY-MM-DD') + interval '1 day'", - new String[]{"2014-01-05 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-05 00:00:00"}); testSimpleEval("SELECT to_date('201404', 'yyyymm');", new String[]{"2014-04-01"}); } @@ -326,106 +366,110 @@ public class TestDateTimeFunctions extends ExprTestBase { @Test public void testAddMonths() throws Exception { testSimpleEval("SELECT add_months(date '2013-12-17', 2::INT2);", - new String[]{"2014-02-17 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 00:00:00"}); testSimpleEval("SELECT add_months(date '2013-12-17', 2::INT4);", - new String[]{"2014-02-17 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 00:00:00"}); testSimpleEval("SELECT add_months(date '2013-12-17', 2::INT8);", - new String[]{"2014-02-17 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 00:00:00"}); testSimpleEval("SELECT add_months(timestamp '2013-12-17 12:10:20', 2::INT2);", - new String[]{"2014-02-17 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 12:10:20"}); testSimpleEval("SELECT add_months(timestamp '2013-12-17 12:10:20', 2::INT4);", - new String[]{"2014-02-17 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 12:10:20"}); testSimpleEval("SELECT add_months(timestamp '2013-12-17 12:10:20', 2::INT8);", - new String[]{"2014-02-17 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-02-17 12:10:20"}); testSimpleEval("SELECT add_months(date '2014-02-05', -3::INT2);", - new String[]{"2013-11-05 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 00:00:00"}); testSimpleEval("SELECT add_months(date '2014-02-05', -3::INT4);", - new String[]{"2013-11-05 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 00:00:00"}); testSimpleEval("SELECT add_months(date '2014-02-05', -3::INT8);", - new String[]{"2013-11-05 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 00:00:00"}); testSimpleEval("SELECT add_months(timestamp '2014-02-05 12:10:20', -3::INT2);", - new String[]{"2013-11-05 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 12:10:20"}); testSimpleEval("SELECT add_months(timestamp '2014-02-05 12:10:20', -3::INT4);", - new String[]{"2013-11-05 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 12:10:20"}); testSimpleEval("SELECT add_months(timestamp '2014-02-05 12:10:20', -3::INT8);", - new String[]{"2013-11-05 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-05 12:10:20"}); } @Test public void testAddDays() throws IOException { testSimpleEval("SELECT add_days(date '2013-12-30', 5::INT2);", - new String[]{"2014-01-04 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 00:00:00"}); testSimpleEval("SELECT add_days(date '2013-12-30', 5::INT4);", - new String[]{"2014-01-04 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 00:00:00"}); testSimpleEval("SELECT add_days(date '2013-12-30', 5::INT8);", - new String[]{"2014-01-04 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 00:00:00"}); testSimpleEval("SELECT add_days(timestamp '2013-12-30 12:10:20', 5::INT2);", - new String[]{"2014-01-04 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 12:10:20"}); testSimpleEval("SELECT add_days(timestamp '2013-12-30 12:10:20', 5::INT4);", - new String[]{"2014-01-04 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 12:10:20"}); testSimpleEval("SELECT add_days(timestamp '2013-12-30 12:10:20', 5::INT8);", - new String[]{"2014-01-04 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2014-01-04 12:10:20"}); testSimpleEval("SELECT add_days(date '2013-12-05', -7::INT2);", - new String[]{"2013-11-28 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 00:00:00"}); testSimpleEval("SELECT add_days(date '2013-12-05', -7::INT4);", - new String[]{"2013-11-28 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 00:00:00"}); testSimpleEval("SELECT add_days(date '2013-12-05', -7::INT8);", - new String[]{"2013-11-28 00:00:00" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 00:00:00"}); testSimpleEval("SELECT add_days(timestamp '2013-12-05 12:10:20', -7::INT2);", - new String[]{"2013-11-28 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 12:10:20"}); testSimpleEval("SELECT add_days(timestamp '2013-12-05 12:10:20', -7::INT4);", - new String[]{"2013-11-28 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 12:10:20"}); testSimpleEval("SELECT add_days(timestamp '2013-12-05 12:10:20', -7::INT8);", - new String[]{"2013-11-28 12:10:20" + getUserTimeZoneDisplay()}); + new String[]{"2013-11-28 12:10:20"}); } @Test public void testDateTimeNow() throws IOException { - TimeZone originTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("GMT-6")); - TimeZone systemOriginTimeZone = TimeZone.getDefault(); + TimeZone originalTimezone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("GMT-6")); + + QueryContext context = new QueryContext(getConf()); + context.put(SessionVars.TZ, "GMT-6"); + try { Date expectedDate = new Date(System.currentTimeMillis()); - testSimpleEval("select to_char(now(), 'yyyy-MM-dd');", + testSimpleEval(context, "select to_char(now(), 'yyyy-MM-dd');", new String[]{dateFormat(expectedDate, "yyyy-MM-dd")}); - testSimpleEval("select cast(extract(year from now()) as INT4);", + testSimpleEval(context, "select cast(extract(year from now()) as INT4);", new String[]{dateFormat(expectedDate, "yyyy")}); - testSimpleEval("select current_date();", + testSimpleEval(context, "select current_date();", new String[]{dateFormat(expectedDate, "yyyy-MM-dd")}); - testSimpleEval("select cast(extract(hour from current_time()) as INT4);", + testSimpleEval(context, "select cast(extract(hour from current_time()) as INT4);", new String[]{String.valueOf(Integer.parseInt(dateFormat(expectedDate, "HH")))}); } finally { - TajoConf.setCurrentTimeZone(originTimeZone); - TimeZone.setDefault(systemOriginTimeZone); + TimeZone.setDefault(originalTimezone); } } @Test public void testTimeValueKeyword() throws IOException { - TimeZone originTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("GMT-6")); - TimeZone systemOriginTimeZone = TimeZone.getDefault(); + TimeZone originTimeZone = TimeZone.getDefault(); TimeZone.setDefault(TimeZone.getTimeZone("GMT-6")); + + QueryContext context = new QueryContext(getConf()); + context.put(SessionVars.TZ, "GMT-6"); + try { Date expectedDate = new Date(System.currentTimeMillis()); - testSimpleEval("select to_char(current_timestamp, 'yyyy-MM-dd');", + testSimpleEval(context, "select to_char(current_timestamp, 'yyyy-MM-dd');", new String[]{dateFormat(expectedDate, "yyyy-MM-dd")}); - testSimpleEval("select cast(extract(year from current_timestamp) as INT4);", + testSimpleEval(context, "select cast(extract(year from current_timestamp) as INT4);", new String[]{dateFormat(expectedDate, "yyyy")}); - testSimpleEval("select current_date;", + testSimpleEval(context, "select current_date;", new String[]{dateFormat(expectedDate, "yyyy-MM-dd")}); - testSimpleEval("select cast(extract(hour from current_time) as INT4);", + testSimpleEval(context, "select cast(extract(hour from current_time) as INT4);", new String[]{String.valueOf(Integer.parseInt(dateFormat(expectedDate, "HH")))}); } finally { - TajoConf.setCurrentTimeZone(originTimeZone); - TimeZone.setDefault(systemOriginTimeZone); + TimeZone.setDefault(originTimeZone); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 5e6d05b..5b7641a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -18,11 +18,9 @@ package org.apache.tajo.engine.query; -import org.apache.tajo.IntegrationTest; -import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.TajoConstants; +import com.google.common.collect.Lists; +import org.apache.tajo.*; import org.apache.tajo.TajoProtos.QueryState; -import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; @@ -37,6 +35,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.junit.Assert.*; @@ -538,4 +538,46 @@ public class TestSelectQuery extends QueryTestCaseBase { assertResultSet(res); cleanupQuery(res); } + + @Test + public void testTimezonedTable1() throws Exception { + try { + executeDDL("datetime_table_ddl.sql", "timezoned", new String[]{"timezoned1"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned1"); + } + } + + @Test + public void testTimezonedTable2() throws Exception { + try { + executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned2"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned2"); + } + } + + @Test + public void testTimezonedTable3() throws Exception { + Map<String,String> sessionVars = new HashMap<String, String>(); + sessionVars.put(SessionVars.TZ.name(), "GMT+9"); + getClient().updateSessionVariables(sessionVars); + + try { + executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned3"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned3"); + } + + getClient().unsetSessionVariables(Lists.newArrayList("TZ")); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java index 37454e6..48a1464 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSortQuery.java @@ -125,21 +125,14 @@ public class TestSortQuery extends QueryTestCaseBase { public final void testSortWithDate() throws Exception { // skip this test if catalog uses HCatalogStore. // It is because HCatalogStore does not support Time data type. - TimeZone oldTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC")); - TimeZone systemOldTimeZone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - try { - if (!testingCluster.isHCatalogStoreRunning()) { - // create external table table1 (col1 timestamp, col2 date, col3 time) ... - executeDDL("create_table_with_date_ddl.sql", "table1"); - - ResultSet res = executeQuery(); - assertResultSet(res); - cleanupQuery(res); - } - } finally { - TajoConf.setCurrentTimeZone(oldTimeZone); - TimeZone.setDefault(systemOldTimeZone); + + if (!testingCluster.isHCatalogStoreRunning()) { + // create external table table1 (col1 timestamp, col2 date, col3 time) ... + executeDDL("create_table_with_date_ddl.sql", "table1"); + + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index f491c63..543c17a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -32,6 +32,7 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.client.QueryClientImpl; import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; @@ -105,8 +106,8 @@ public class TestResultSet { } @Test - public void test() throws IOException, SQLException { - TajoResultSet rs = new TajoResultSet(null, null, conf, desc); + public void test() throws Exception { + TajoResultSet rs = new TajoResultSet(TajoTestingCluster.newTajoClient(), null, conf, desc); ResultSetMetaData meta = rs.getMetaData(); assertNotNull(meta); Schema schema = scoreSchema; @@ -133,12 +134,6 @@ public class TestResultSet { // Hcatalog does not support date type, time type in hive-0.12.0 if(util.isHCatalogStoreRunning()) return; - TimeZone tajoCurrentTimeZone = TajoConf.getCurrentTimeZone(); - TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC")); - - TimeZone systemCurrentTimeZone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - ResultSet res = null; TajoClient client = TajoTestingCluster.newTajoClient(); try { @@ -212,8 +207,6 @@ public class TestResultSet { assertNotNull(timestamp); assertEquals("2014-01-01 10:00:00.0", timestamp.toString()); } finally { - TajoConf.setCurrentTimeZone(tajoCurrentTimeZone); - TimeZone.setDefault(systemCurrentTimeZone); if (res != null) { res.close(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index a004baa..99baeba 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -543,9 +543,6 @@ public class TestTajoJdbc extends QueryTestCaseBase { // skip this test if catalog uses HCatalogStore. // It is because HCatalogStore does not support Time data type. - TimeZone oldTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC")); - TimeZone systemOldTimeZone = TimeZone.getDefault(); - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); try { if (!testingCluster.isHCatalogStoreRunning()) { @@ -575,9 +572,6 @@ public class TestTajoJdbc extends QueryTestCaseBase { } } finally { - TajoConf.setCurrentTimeZone(oldTimeZone); - TimeZone.setDefault(systemOldTimeZone); - cleanupQuery(res); if (stmt != null) { stmt.close(); http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl b/tajo-core/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl new file mode 100644 index 0000000..38e8bd9 --- /dev/null +++ b/tajo-core/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl @@ -0,0 +1,3 @@ +1980-4-1 01:50:30.010|1980-04-01 +80/4/1 1:50:30 AM|80/4/1 +1980 April 1 1:50:30|1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql new file mode 100644 index 0000000..beb19b7 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql @@ -0,0 +1,4 @@ +CREATE EXTERNAL TABLE ${0} ( + t_timestamp TIMESTAMP, + t_date DATE +) USING TEXTFILE LOCATION ${table.path} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql new file mode 100644 index 0000000..11014d8 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql @@ -0,0 +1,4 @@ +CREATE EXTERNAL TABLE ${0} ( + t_timestamp TIMESTAMP, + t_date DATE +) USING TEXTFILE WITH ('timezone' = 'GMT+9') LOCATION ${table.path} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable1.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable1.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable1.sql new file mode 100644 index 0000000..38c9e90 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable1.sql @@ -0,0 +1 @@ +SELECT * FROM timezoned1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql new file mode 100644 index 0000000..722fc65 --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable2.sql @@ -0,0 +1 @@ +SELECT * FROM timezoned2; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable3.sql ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable3.sql b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable3.sql new file mode 100644 index 0000000..32b9c3a --- /dev/null +++ b/tajo-core/src/test/resources/queries/TestSelectQuery/testTimezonedTable3.sql @@ -0,0 +1 @@ +SELECT * FROM timezoned3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result new file mode 100644 index 0000000..39f593b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result @@ -0,0 +1,5 @@ +t_timestamp,t_date +------------------------------- +1980-04-01 01:50:30.01,1980-04-01 +1980-04-01 01:50:30,1980-04-01 +1980-04-01 01:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result new file mode 100644 index 0000000..916f4be --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result @@ -0,0 +1,5 @@ +t_timestamp,t_date +------------------------------- +1980-03-31 16:50:30.01,1980-04-01 +1980-03-31 16:50:30,1980-04-01 +1980-03-31 16:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result new file mode 100644 index 0000000..39f593b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result @@ -0,0 +1,5 @@ +t_timestamp,t_date +------------------------------- +1980-04-01 01:50:30.01,1980-04-01 +1980-04-01 01:50:30,1980-04-01 +1980-04-01 01:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-docs/src/main/sphinx/table_management.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/table_management.rst b/tajo-docs/src/main/sphinx/table_management.rst index 62a8e45..2b21ddc 100644 --- a/tajo-docs/src/main/sphinx/table_management.rst +++ b/tajo-docs/src/main/sphinx/table_management.rst @@ -2,8 +2,11 @@ Table Management ****************** +In Tajo, a table is a logical view of one data sources. Logically, one table consists of a logical schema, partitions, URL, and various properties. Physically, A table can be a directory in HDFS, a single file, one HBase table, or a RDBMS table. In order to make good use of Tajo, users need to understand features and physical characteristics of their physical layout. This section explains all about table management. + .. toctree:: :maxdepth: 1 + table_management/table_overview table_management/file_formats - table_management/compression + table_management/compression \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-docs/src/main/sphinx/table_management/csv.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/table_management/csv.rst b/tajo-docs/src/main/sphinx/table_management/csv.rst index 71313d6..37dc2ba 100644 --- a/tajo-docs/src/main/sphinx/table_management/csv.rst +++ b/tajo-docs/src/main/sphinx/table_management/csv.rst @@ -39,9 +39,9 @@ Now, the CSV storage format provides the following physical properties. * ``text.delimiter``: delimiter character. ``|`` or ``\u0001`` is usually used, and the default field delimiter is ``|``. * ``text.null``: NULL character. The default NULL character is an empty string ``''``. Hive's default NULL character is ``'\\N'``. * ``compression.codec``: Compression codec. You can enable compression feature and set specified compression algorithm. The compression algorithm used to compress files. The compression codec name should be the fully qualified class name inherited from `org.apache.hadoop.io.compress.CompressionCodec <https://hadoop.apache.org/docs/current/api/org/apache/hadoop/io/compress/CompressionCodec.html>`_. By default, compression is disabled. -* ``csvfile.serde``: custom (De)serializer class. ``org.apache.tajo.storage.TextSerializerDeserializer`` is the default (De)serializer class. +* ``csvfile.serde`` (deprecated): custom (De)serializer class. ``org.apache.tajo.storage.TextSerializerDeserializer`` is the default (De)serializer class. +* ``timezone``: the time zone that the table uses for writting. When table rows are read or written, ```timestamp``` and ```time``` column values are adjusted by this timezone if it is set. Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'UTC+9' or a location-based form like 'Asia/Seoul'. * ``text.error-tolerance.max-num``: the maximum number of permissible parsing errors. This value should be an integer value. By default, ``text.error-tolerance.max-num`` is ``0``. According to the value, parsing errors will be handled in different ways. - * If ``text.error-tolerance.max-num < 0``, all parsing errors are ignored. * If ``text.error-tolerance.max-num == 0``, any parsing error is not allowed. If any error occurs, the query will be failed. (default) * If ``text.error-tolerance.max-num > 0``, the given number of parsing errors in each task will be pemissible. http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-docs/src/main/sphinx/table_management/table_overview.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/table_management/table_overview.rst b/tajo-docs/src/main/sphinx/table_management/table_overview.rst new file mode 100644 index 0000000..bb4b827 --- /dev/null +++ b/tajo-docs/src/main/sphinx/table_management/table_overview.rst @@ -0,0 +1,98 @@ +************************************* +Overview of Tajo Tables +************************************* + +Overview +======== + +.. todo:: + +Table Properties +================ +All table formats provide parameters for enabling or disabling features and adjusting physical parameters. +The ``WITH`` clause in the CREATE TABLE statement allows users to set those properties. + +The following example is to set a custom field delimiter, NULL character, and compression codec: + +.. code-block:: sql + + CREATE TABLE table1 ( + id int, + name text, + score float, + type text + ) USING CSV WITH('text.delimiter'='\u0001', + 'text.null'='\\N', + 'compression.codec'='org.apache.hadoop.io.compress.SnappyCodec'); + +Each physical table layout has its own specialized properties. They will be addressed in :doc:`/table_management/file_formats`. + + +Common Table Properties +======================= + +There are some common table properties which are used in most tables. + +Compression +----------- +.. todo:: + +Time zone +--------- +In Tajo, a table property ``timezone`` allows users to specify a time zone that the table uses for reading or writing. +When each table row are read or written, ```timestamp``` and ```time``` column values are adjusted by a given time zone if it is set. Time zone can be an abbreviation form like 'PST' or 'DST'. Also, it accepts an offset-based form like 'GMT+9' or UTC+9' or a location-based form like 'Asia/Seoul'. + +Each table has one time zone, and many tables can have different time zones. Internally, Tajo translates all tables data to offset-based values. So, complex queries like join with multiple time zones work well. + +.. note:: + + In many cases, offset-based forms or locaion-based forms are recommanded. In order to know the list of time zones, please refer to `List of tz database time zones <http://en.wikipedia.org/wiki/List_of_tz_database_time_zones>`_ + +.. note:: + + Java 6 does not recognize many location-based time zones and an offset-based timezone using the prefix 'UTC'. We highly recommanded using the offset-based time zone using the prefix 'GMT'. In other words, you should use 'GMT-7' instead of 'UTC-7' in Java 6. + +How time zone works in Tajo +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For example, consider that there is a list of delimited text lines where each rows are written with ``Asia/Seoul`` time zone (i.e., GMT + 9). + +.. code-block:: text + + 1980-4-1 01:50:30.010|1980-04-01 + 80/4/1 1:50:30 AM|80/4/1 + 1980 April 1 1:50:30|1980-04-01 + + +In order to register the table, we should put a table property ``'timezone'='Asia/Seoul'`` in ``CREATE TABLE`` statement as follows: + +.. code-block:: sql + + CREATE EXTERNAL TABLE table1 ( + t_timestamp TIMESTAMP, + t_date DATE + ) USING TEXTFILE WITH('text.delimiter'='|', 'timezone'='ASIA/Seoul') LOCATION '/path-to-table/' + + +By default, ``tsql`` and ``TajoClient`` API use UTC time zone. So, timestamp values in the result are adjusted by the time zone offset. But, date is not adjusted because date type does not consider time zone. + +.. code-block:: sql + + default> SELECT * FROM table1 + t_timestamp, t_date + ---------------------------------- + 1980-03-31 16:50:30.01, 1980-04-01 + 1980-03-31 16:50:30 , 1980-04-01 + 1980-03-31 16:50:30 , 1980-04-01 + +In addition, users can set client-side time zone by setting a session variable 'TZ'. It enables a client to translate timestamp or time values to user's time zoned ones. + +.. code-block:: sql + + default> \set TZ 'Asia/Seoul' + default> SELECT * FROM table1 + t_timestamp, t_date + ---------------------------------- + 1980-04-01 01:50:30.01, 1980-04-01 + 1980-04-01 01:50:30 , 1980-04-01 + 1980-04-01 01:50:30 , 1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java ---------------------------------------------------------------------- diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java index faa058d..8f5bed6 100644 --- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java +++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoMetaDataResultSet.java @@ -31,12 +31,14 @@ public class TajoMetaDataResultSet extends TajoResultSetBase { private List<MetaDataTuple> values; public TajoMetaDataResultSet(Schema schema, List<MetaDataTuple> values) { + super(null); init(); this.schema = schema; setDataTuples(values); } public TajoMetaDataResultSet(List<String> columns, List<Type> types, List<MetaDataTuple> values) { + super(null); init(); schema = new Schema(); http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index 406cdfc..62ba33c 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -19,6 +19,8 @@ package org.apache.tajo.plan; import com.google.common.collect.Sets; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; @@ -43,6 +45,7 @@ import org.apache.tajo.util.datetime.TimeMeta; import java.util.Set; import java.util.Stack; +import java.util.TimeZone; import static org.apache.tajo.algebra.WindowSpec.WindowFrameEndBoundType; import static org.apache.tajo.algebra.WindowSpec.WindowFrameStartBoundType; @@ -64,11 +67,16 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva } static class Context { + OverridableConf queryContext; + TimeZone timeZone; LogicalPlan plan; LogicalPlan.QueryBlock currentBlock; NameResolvingMode columnRsvLevel; public Context(LogicalPlanner.PlanContext planContext, NameResolvingMode colRsvLevel) { + this.queryContext = planContext.queryContext; + this.timeZone = planContext.timeZone; + this.plan = planContext.plan; this.currentBlock = planContext.queryBlock; this.columnRsvLevel = colRsvLevel; @@ -95,7 +103,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva * @param rhs right hand side term * @return a pair including left/right hand side terms */ - public static Pair<EvalNode, EvalNode> convertTypesIfNecessary(EvalNode lhs, EvalNode rhs) { + private static Pair<EvalNode, EvalNode> convertTypesIfNecessary(Context ctx, EvalNode lhs, EvalNode rhs) { Type lhsType = lhs.getValueType().getType(); Type rhsType = rhs.getValueType().getType(); @@ -108,10 +116,10 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva if (toBeCasted != null) { // if not null, one of either should be converted to another type. // Overwrite lhs, rhs, or both with cast expression. if (lhsType != toBeCasted) { - lhs = convertType(lhs, CatalogUtil.newSimpleDataType(toBeCasted)); + lhs = convertType(ctx, lhs, CatalogUtil.newSimpleDataType(toBeCasted)); } if (rhsType != toBeCasted) { - rhs = convertType(rhs, CatalogUtil.newSimpleDataType(toBeCasted)); + rhs = convertType(ctx, rhs, CatalogUtil.newSimpleDataType(toBeCasted)); } } @@ -126,7 +134,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva * @param toType target type * @return type converted expression. */ - private static EvalNode convertType(EvalNode evalNode, DataType toType) { + private static EvalNode convertType(Context ctx, EvalNode evalNode, DataType toType) { // if original and toType is the same, we don't need type conversion. if (evalNode.getValueType().equals(toType)) { @@ -140,9 +148,9 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva if (evalNode.getType() == EvalType.BETWEEN) { BetweenPredicateEval between = (BetweenPredicateEval) evalNode; - between.setPredicand(convertType(between.getPredicand(), toType)); - between.setBegin(convertType(between.getBegin(), toType)); - between.setEnd(convertType(between.getEnd(), toType)); + between.setPredicand(convertType(ctx, between.getPredicand(), toType)); + between.setBegin(convertType(ctx, between.getBegin(), toType)); + between.setEnd(convertType(ctx, between.getEnd(), toType)); return between; @@ -150,11 +158,11 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva CaseWhenEval caseWhenEval = (CaseWhenEval) evalNode; for (CaseWhenEval.IfThenEval ifThen : caseWhenEval.getIfThenEvals()) { - ifThen.setResult(convertType(ifThen.getResult(), toType)); + ifThen.setResult(convertType(ctx, ifThen.getResult(), toType)); } if (caseWhenEval.hasElse()) { - caseWhenEval.setElseResult(convertType(caseWhenEval.getElse(), toType)); + caseWhenEval.setElseResult(convertType(ctx, caseWhenEval.getElse(), toType)); } return caseWhenEval; @@ -166,7 +174,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva Datum[] convertedDatum = new Datum[datums.length]; for (int i = 0; i < datums.length; i++) { - convertedDatum[i] = DatumFactory.cast(datums[i], toType); + convertedDatum[i] = DatumFactory.cast(datums[i], toType, ctx.timeZone); } RowConstantEval convertedRowConstant = new RowConstantEval(convertedDatum); @@ -175,11 +183,11 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva } else if (evalNode.getType() == EvalType.CONST) { ConstEval original = (ConstEval) evalNode; - ConstEval newConst = new ConstEval(DatumFactory.cast(original.getValue(), toType)); + ConstEval newConst = new ConstEval(DatumFactory.cast(original.getValue(), toType, ctx.timeZone)); return newConst; } else { - return new CastEval(evalNode, toType); + return new CastEval(ctx.queryContext, evalNode, toType); } } @@ -279,7 +287,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva throw new IllegalStateException("Wrong Expr Type: " + expr.getType()); } - return createBinaryNode(evalType, left, right); + return createBinaryNode(ctx, evalType, left, right); } /////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -308,7 +316,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva between.isSymmetric(), predicand, begin, end); - betweenEval = (BetweenPredicateEval) convertType(betweenEval, widestType); + betweenEval = (BetweenPredicateEval) convertType(ctx, betweenEval, widestType); return betweenEval; } @@ -342,7 +350,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva assertEval(widestType != null, "Invalid Type Conversion for CaseWhen"); // implicit type conversion - caseWhenEval = (CaseWhenEval) convertType(caseWhenEval, widestType); + caseWhenEval = (CaseWhenEval) convertType(ctx, caseWhenEval, widestType); return caseWhenEval; } @@ -362,7 +370,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva RowConstantEval rowConstantEval = (RowConstantEval) visit(ctx, stack, expr.getInValue()); stack.pop(); - Pair<EvalNode, EvalNode> pair = convertTypesIfNecessary(lhs, rowConstantEval); + Pair<EvalNode, EvalNode> pair = convertTypesIfNecessary(ctx, lhs, rowConstantEval); return new InEval(pair.getFirst(), (RowConstantEval) pair.getSecond(), expr.isNot()); } @@ -415,10 +423,10 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva stack.pop(); if (lhs.getValueType().getType() != Type.TEXT) { - lhs = convertType(lhs, CatalogUtil.newSimpleDataType(Type.TEXT)); + lhs = convertType(ctx, lhs, CatalogUtil.newSimpleDataType(Type.TEXT)); } if (rhs.getValueType().getType() != Type.TEXT) { - rhs = convertType(rhs, CatalogUtil.newSimpleDataType(Type.TEXT)); + rhs = convertType(ctx, rhs, CatalogUtil.newSimpleDataType(Type.TEXT)); } return new BinaryEval(EvalType.CONCATENATE, lhs, rhs); @@ -448,8 +456,8 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva // Arithmetic Operators /////////////////////////////////////////////////////////////////////////////////////////////////////////// - private static BinaryEval createBinaryNode(EvalType type, EvalNode lhs, EvalNode rhs) { - Pair<EvalNode, EvalNode> pair = convertTypesIfNecessary(lhs, rhs); // implicit type conversion if necessary + private static BinaryEval createBinaryNode(Context ctx, EvalType type, EvalNode lhs, EvalNode rhs) { + Pair<EvalNode, EvalNode> pair = convertTypesIfNecessary(ctx, lhs, rhs); // implicit type conversion if necessary return new BinaryEval(type, pair.getFirst(), pair.getSecond()); } @@ -460,7 +468,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.PLUS, left, right); + return createBinaryNode(ctx, EvalType.PLUS, left, right); } @Override @@ -470,7 +478,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.MINUS, left, right); + return createBinaryNode(ctx, EvalType.MINUS, left, right); } @Override @@ -480,7 +488,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.MULTIPLY, left, right); + return createBinaryNode(ctx, EvalType.MULTIPLY, left, right); } @Override @@ -490,7 +498,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.DIVIDE, left, right); + return createBinaryNode(ctx, EvalType.DIVIDE, left, right); } @Override @@ -500,7 +508,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva EvalNode right = visit(ctx, stack, expr.getRight()); stack.pop(); - return createBinaryNode(EvalType.MODULAR, left, right); + return createBinaryNode(ctx, EvalType.MODULAR, left, right); } /////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -584,14 +592,14 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva } else { lastDataType = CatalogUtil.newSimpleDataType(CatalogUtil.getPrimitiveTypeOf(lastDataType.getType())); } - givenArgs[i] = convertType(givenArgs[i], lastDataType); + givenArgs[i] = convertType(ctx, givenArgs[i], lastDataType); } } else { assertEval(funcDesc.getParamTypes().length == givenArgs.length, "The number of parameters is mismatched to the function definition: " + funcDesc.toString()); // According to our function matching method, each given argument can be casted to the definition parameter. for (int i = 0; i < givenArgs.length; i++) { - givenArgs[i] = convertType(givenArgs[i], funcDesc.getParamTypes()[i]); + givenArgs[i] = convertType(ctx, givenArgs[i], funcDesc.getParamTypes()[i]); } } @@ -600,7 +608,7 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva FunctionType functionType = funcDesc.getFuncType(); if (functionType == FunctionType.GENERAL || functionType == FunctionType.UDF) { - return new GeneralFunctionEval(funcDesc, (GeneralFunction) funcDesc.newInstance(), givenArgs); + return new GeneralFunctionEval(ctx.queryContext, funcDesc, (GeneralFunction) funcDesc.newInstance(), givenArgs); } else if (functionType == FunctionType.AGGREGATION || functionType == FunctionType.UDA) { if (!ctx.currentBlock.hasNode(NodeType.GROUP_BY)) { @@ -768,11 +776,23 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva public EvalNode visitCastExpr(Context ctx, Stack<Expr> stack, CastExpr expr) throws PlanningException { EvalNode child = super.visitCastExpr(ctx, stack, expr); - if (child.getType() == EvalType.CONST) { // if it is a casting operation for a constant value - ConstEval constEval = (ConstEval) child; // it will be pre-computed and casted to a constant value - return new ConstEval(DatumFactory.cast(constEval.getValue(), LogicalPlanner.convertDataType(expr.getTarget()))); + // if it is a casting operation for a constant value, it will be pre-computed and casted to a constant value. + + if (child.getType() == EvalType.CONST) { + ConstEval constEval = (ConstEval) child; + + // some cast operation may require earlier evaluation with timezone. + TimeZone tz = null; + if (ctx.queryContext.containsKey(SessionVars.TZ)) { + String tzId = ctx.queryContext.get(SessionVars.TZ); + tz = TimeZone.getTimeZone(tzId); + } + + return new ConstEval( + DatumFactory.cast(constEval.getValue(), LogicalPlanner.convertDataType(expr.getTarget()), tz)); + } else { - return new CastEval(child, LogicalPlanner.convertDataType(expr.getTarget())); + return new CastEval(ctx.queryContext, child, LogicalPlanner.convertDataType(expr.getTarget())); } } @@ -838,7 +858,11 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva TimeMeta tm = new TimeMeta(); DateTimeUtil.toJulianTimeMeta(timestamp, tm); - DateTimeUtil.toUTCTimezone(tm); + + if (ctx.queryContext.containsKey(SessionVars.TZ)) { + TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TZ)); + DateTimeUtil.toUTCTimezone(tm, tz); + } return new ConstEval(new TimestampDatum(DateTimeUtil.toJulianTimestamp(tm))); } @@ -864,7 +888,11 @@ public class ExprAnnotator extends BaseAlgebraVisitor<ExprAnnotator.Context, Eva } TimeDatum timeDatum = new TimeDatum(time); TimeMeta tm = timeDatum.toTimeMeta(); - DateTimeUtil.toUTCTimezone(tm); + + if (ctx.queryContext.containsKey(SessionVars.TZ)) { + TimeZone tz = TimeZone.getTimeZone(ctx.queryContext.get(SessionVars.TZ)); + DateTimeUtil.toUTCTimezone(tm, tz); + } return new ConstEval(new TimeDatum(DateTimeUtil.toTime(tm))); } http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 69c0e4b..f21bbb5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -83,6 +83,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex LogicalPlan plan; QueryBlock queryBlock; EvalTreeOptimizer evalOptimizer; + TimeZone timeZone; boolean debugOrUnitTests; public PlanContext(OverridableConf context, LogicalPlan plan, QueryBlock block, EvalTreeOptimizer evalOptimizer, @@ -91,6 +92,13 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex this.plan = plan; this.queryBlock = block; this.evalOptimizer = evalOptimizer; + + // session's time zone + if (context.containsKey(SessionVars.TZ)) { + String timezoneId = context.get(SessionVars.TZ); + timeZone = TimeZone.getTimeZone(timezoneId); + } + this.debugOrUnitTests = debugOrUnitTests; } http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java index d625a11..700913e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java @@ -19,19 +19,31 @@ package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Schema; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.TUtil; + +import java.util.TimeZone; import static org.apache.tajo.common.TajoDataTypes.DataType; public class CastEval extends UnaryEval { @Expose private DataType target; + @Expose private TimeZone timezone; - public CastEval(EvalNode operand, DataType target) { + public CastEval(OverridableConf context, EvalNode operand, DataType target) { super(EvalType.CAST, operand); this.target = target; + + if (context.containsKey(SessionVars.TZ)) { + String timezoneId = context.get(SessionVars.TZ); + timezone = TimeZone.getTimeZone(timezoneId); + } } public EvalNode getOperand() { @@ -43,6 +55,14 @@ public class CastEval extends UnaryEval { return target; } + public boolean hasTimeZone() { + return this.timezone != null; + } + + public TimeZone getTimezone() { + return this.timezone; + } + @Override public String getName() { return target.getType().name(); @@ -54,7 +74,7 @@ public class CastEval extends UnaryEval { return operandDatum; } - return DatumFactory.cast(operandDatum, target); + return DatumFactory.cast(operandDatum, target, timezone); } public String toString() { @@ -66,7 +86,9 @@ public class CastEval extends UnaryEval { boolean valid = obj != null && obj instanceof CastEval; if (valid) { CastEval another = (CastEval) obj; - return child.equals(another.child) && target.equals(another.target); + return child.equals(another.child) && + target.equals(another.target) && + TUtil.checkEquals(timezone, another.timezone); } else { return false; } http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java index e28e5f3..0c6f1e2 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java @@ -20,6 +20,7 @@ package org.apache.tajo.plan.expr; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.Schema; import org.apache.tajo.datum.Datum; @@ -28,14 +29,17 @@ import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.TUtil; +import javax.annotation.Nullable; + public class GeneralFunctionEval extends FunctionEval { @Expose protected GeneralFunction instance; private Tuple params = null; - public GeneralFunctionEval(FunctionDesc desc, GeneralFunction instance, EvalNode[] givenArgs) { + public GeneralFunctionEval(@Nullable OverridableConf queryContext, FunctionDesc desc, GeneralFunction instance, + EvalNode[] givenArgs) { super(EvalType.FUNCTION, desc, givenArgs); this.instance = instance; - this.instance.init(getParamType()); + this.instance.init(queryContext, getParamType()); } /* (non-Javadoc) http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java index 006449f..39db5c6 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java @@ -18,6 +18,7 @@ package org.apache.tajo.plan.function; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -36,7 +37,8 @@ public abstract class GeneralFunction extends Function implements GsonObject { /** * This method gives hints to an actual function instance. */ - public void init(FunctionEval.ParamType [] paramTypes) {} + @SuppressWarnings("unused") + public void init(OverridableConf queryContext, FunctionEval.ParamType [] paramTypes) {} public abstract Datum eval(Tuple params); http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java index e6d54b1..89b4fc0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java @@ -20,6 +20,7 @@ package org.apache.tajo.plan.serder; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.exception.NoSuchFunctionException; @@ -43,7 +44,7 @@ import java.util.*; */ public class EvalTreeProtoDeserializer { - public static EvalNode deserialize(PlanProto.EvalTree tree) { + public static EvalNode deserialize(OverridableConf context, PlanProto.EvalTree tree) { Map<Integer, EvalNode> evalNodeMap = Maps.newHashMap(); // sort serialized eval nodes in an ascending order of their IDs. @@ -79,7 +80,7 @@ public class EvalTreeProtoDeserializer { current = new IsNullEval(unaryProto.getNegative(), child); break; case CAST: - current = new CastEval(child, unaryProto.getCastingType()); + current = new CastEval(context, child, unaryProto.getCastingType()); break; case SIGNED: current = new SignedEval(unaryProto.getNegative(), child); @@ -153,7 +154,7 @@ public class EvalTreeProtoDeserializer { funcDesc = new FunctionDesc(funcProto.getFuncion()); if (type == EvalType.FUNCTION) { GeneralFunction instance = (GeneralFunction) funcDesc.newInstance(); - current = new GeneralFunctionEval(new FunctionDesc(funcProto.getFuncion()), instance, params); + current = new GeneralFunctionEval(context, new FunctionDesc(funcProto.getFuncion()), instance, params); } else if (type == EvalType.AGG_FUNCTION || type == EvalType.WINDOW_FUNCTION) { AggFunction instance = (AggFunction) funcDesc.newInstance(); if (type == EvalType.AGG_FUNCTION) { http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java index 9f22c20..92a245f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java @@ -111,6 +111,9 @@ public class EvalTreeProtoSerializer } else if (unary.getType() == EvalType.CAST) { CastEval castEval = (CastEval) unary; unaryBuilder.setCastingType(castEval.getValueType()); + if (castEval.hasTimeZone()) { + unaryBuilder.setTimezone(castEval.getTimezone().getID()); + } } // registering itself and building EvalNode http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java index 8cafbd0..befeb2a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java @@ -36,6 +36,7 @@ import org.apache.tajo.util.TUtil; import java.lang.reflect.Type; import java.util.Map; +import java.util.TimeZone; public class PlanGsonHelper { private static Gson gson; @@ -56,6 +57,7 @@ public class PlanGsonHelper { adapters.put(AggFunction.class, new FunctionAdapter()); adapters.put(Datum.class, new DatumAdapter()); adapters.put(DataType.class, new DataTypeAdapter()); + adapters.put(TimeZone.class, new TimeZoneGsonSerdeAdapter()); return adapters; } http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index c55c203..8a9e1ca 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -791,6 +791,10 @@ public class PlannerUtil { if (sessionVars.containsKey(SessionVars.NULL_CHAR)) { tableProperties.set(StorageConstants.TEXT_NULL, sessionVars.get(SessionVars.NULL_CHAR)); } + + if (sessionVars.containsKey(SessionVars.TZ)) { + tableProperties.set(StorageConstants.TIMEZONE, sessionVars.get(SessionVars.TZ)); + } } } } http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-plan/src/main/proto/Plan.proto ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto index 8639117..0f82d87 100644 --- a/tajo-plan/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -150,6 +150,7 @@ message UnaryEval { required int32 child_id = 1; optional DataType castingType = 2; optional bool negative = 3; + optional string timezone = 4; } message BinaryEval { http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java index 094d285..ce7b11d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/TextSerializerDeserializer.java @@ -31,14 +31,13 @@ import org.apache.tajo.util.NumberUtil; import java.io.IOException; import java.io.OutputStream; -//Compatibility with Apache Hive +// Compatibility with Apache Hive @Deprecated public class TextSerializerDeserializer implements SerializerDeserializer { public static final byte[] trueBytes = "true".getBytes(); public static final byte[] falseBytes = "false".getBytes(); private ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance(); - @Override public int serialize(Column col, Datum datum, OutputStream out, byte[] nullCharacters) throws IOException { @@ -86,12 +85,12 @@ public class TextSerializerDeserializer implements SerializerDeserializer { out.write(bytes); break; case TIME: - bytes = ((TimeDatum)datum).asChars(TajoConf.getCurrentTimeZone(), true).getBytes(); + bytes = ((TimeDatum)datum).asChars(TajoConf.getSystemTimezone(), true).getBytes(); length = bytes.length; out.write(bytes); break; case TIMESTAMP: - bytes = ((TimestampDatum)datum).asChars(TajoConf.getCurrentTimeZone(), true).getBytes(); + bytes = ((TimestampDatum)datum).asChars(TajoConf.getSystemTimezone(), true).getBytes(); length = bytes.length; out.write(bytes); break; http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java index f2eebc6..1599f62 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java @@ -45,7 +45,7 @@ public class CSVLineDeserializer extends TextLineDeserializer { } nullChars = TextLineSerDe.getNullChars(meta); - fieldSerDer = new TextFieldSerializerDeserializer(); + fieldSerDer = new TextFieldSerializerDeserializer(meta); } public void deserialize(final ByteBuf lineBuf, Tuple output) throws IOException, TextLineParsingError { http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java index 7397000..c0fc18f 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java @@ -45,7 +45,7 @@ public class CSVLineSerializer extends TextLineSerializer { delimiter = CSVLineSerDe.getFieldDelimiter(meta); columnNum = schema.size(); - serde = new TextFieldSerializerDeserializer(); + serde = new TextFieldSerializerDeserializer(meta); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/facd1ddc/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java index 95d0407..ae7565d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java @@ -22,25 +22,36 @@ import com.google.protobuf.Message; import io.netty.buffer.ByteBuf; import io.netty.util.CharsetUtil; import org.apache.commons.codec.binary.Base64; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.*; import org.apache.tajo.datum.protobuf.ProtobufJsonFormat; import org.apache.tajo.storage.FieldSerializerDeserializer; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.NumberUtil; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.CharsetDecoder; +import java.util.TimeZone; -//Compatibility with Apache Hive public class TextFieldSerializerDeserializer implements FieldSerializerDeserializer { public static final byte[] trueBytes = "true".getBytes(); public static final byte[] falseBytes = "false".getBytes(); private static ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance(); private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); + private final boolean hasTimezone; + private final TimeZone timezone; + + public TextFieldSerializerDeserializer(TableMeta meta) { + hasTimezone = meta.containsOption(StorageConstants.TIMEZONE); + timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); + } + private static boolean isNull(ByteBuf val, ByteBuf nullBytes) { return !val.isReadable() || nullBytes.equals(val); } @@ -50,7 +61,8 @@ public class TextFieldSerializerDeserializer implements FieldSerializerDeseriali } @Override - public int serialize(OutputStream out, Datum datum, Column col, int columnIndex, byte[] nullChars) throws IOException { + public int serialize(OutputStream out, Datum datum, Column col, int columnIndex, byte[] nullChars) + throws IOException { byte[] bytes; int length = 0; TajoDataTypes.DataType dataType = col.getDataType(); @@ -95,12 +107,20 @@ public class TextFieldSerializerDeserializer implements FieldSerializerDeseriali out.write(bytes); break; case TIME: - bytes = ((TimeDatum) datum).asChars(TajoConf.getCurrentTimeZone(), true).getBytes(); + if (hasTimezone) { + bytes = ((TimeDatum) datum).asChars(timezone, true).getBytes(); + } else { + bytes = datum.asTextBytes(); + } length = bytes.length; out.write(bytes); break; case TIMESTAMP: - bytes = ((TimestampDatum) datum).asChars(TajoConf.getCurrentTimeZone(), true).getBytes(); + if (hasTimezone) { + bytes = ((TimestampDatum) datum).asChars(timezone, true).getBytes(); + } else { + bytes = datum.asTextBytes(); + } length = bytes.length; out.write(bytes); break; @@ -178,12 +198,22 @@ public class TextFieldSerializerDeserializer implements FieldSerializerDeseriali decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); break; case TIME: - datum = DatumFactory.createTime( - decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); + if (hasTimezone) { + datum = DatumFactory.createTime( + decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), timezone); + } else { + datum = DatumFactory.createTime( + decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); + } break; case TIMESTAMP: - datum = DatumFactory.createTimestamp( - decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); + if (hasTimezone) { + datum = DatumFactory.createTimestamp( + decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString(), timezone); + } else { + datum = DatumFactory.createTimestamp( + decoder.decode(buf.nioBuffer(buf.readerIndex(), buf.readableBytes())).toString()); + } break; case INTERVAL: datum = DatumFactory.createInterval(
