Repository: tajo Updated Branches: refs/heads/master 3a30f45c7 -> 3e8f4a030
TAJO-1812: Timezone support in JSON file format. Closes #723 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3e8f4a03 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3e8f4a03 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3e8f4a03 Branch: refs/heads/master Commit: 3e8f4a0307cb857434e8a0c6415e39453f97cabe Parents: 3a30f45 Author: Hyunsik Choi <[email protected]> Authored: Thu Sep 3 21:50:56 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Thu Sep 3 21:51:46 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../tajo/engine/query/TestJsonWithTimezone.java | 115 +++++++++++++++++++ .../timezoned/TestJsonWithTimezone.json | 3 + .../TestSelectQuery/timezoned/table1.tbl | 6 +- .../TestJsonWithTimezone/datetime_table_ddl.sql | 5 + .../datetime_table_timezoned_ddl.sql | 5 + .../testTimezonedTable1.sql | 1 + .../testTimezonedTable2.sql | 2 + .../testTimezonedTable3.sql | 1 + .../testTimezonedTable4.sql | 2 + .../testTimezonedTable5.sql | 2 + .../TestSelectQuery/datetime_table_ddl.sql | 3 +- .../datetime_table_timezoned_ddl.sql | 3 +- .../datetime_table_timezoned_ddl2.sql | 3 +- .../testTimezonedTable1.result | 5 + .../testTimezonedTable2.result | 5 + .../testTimezonedTable3.result | 5 + .../TestSelectQuery/testTimezonedTable1.result | 8 +- .../TestSelectQuery/testTimezonedTable2.result | 8 +- .../TestSelectQuery/testTimezonedTable3.result | 8 +- .../tajo/storage/json/JsonLineDeserializer.java | 21 +++- .../tajo/storage/json/JsonLineSerializer.java | 35 +++++- 22 files changed, 224 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index d306497..32cf77f 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,8 @@ Release 0.11.0 - unreleased NEW FEATURES + TAJO-1812: Timezone support in JSON file format. (hyunsik) + TAJO-1486: Text file should support to skip header rows when creating external table. (Contributed by Jongyoung Park. Committed by jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java new file mode 100644 index 0000000..e526f5c --- /dev/null +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJsonWithTimezone.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import com.google.common.collect.Lists; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.SessionVars; +import org.junit.Test; + +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +public class TestJsonWithTimezone extends QueryTestCaseBase { + @Test + public void testTimezonedTable1() throws Exception { + // Table - GMT (No table property or no system timezone) + // Client - GMT (default client time zone is used if no TIME ZONE session variable is given.) + try { + executeDDL("datetime_table_ddl.sql", "timezoned", new String[]{"timezoned1"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned1"); + } + } + + @Test + public void testTimezonedTable2() throws Exception { + // Table - timezone = GMT+9 + // Client - GMT (SET TIME ZONE 'GMT';) + try { + executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned2"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned2"); + } + } + + @Test + public void testTimezonedTable3() throws Exception { + // Table - timezone = GMT+9 + // Client - GMT+9 through TajoClient API + + Map<String,String> sessionVars = new HashMap<String, String>(); + sessionVars.put(SessionVars.TIMEZONE.name(), "GMT+9"); + getClient().updateSessionVariables(sessionVars); + + try { + executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned3"}); + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned3"); + } + + getClient().unsetSessionVariables(Lists.newArrayList("TIMEZONE")); + } + + @Test + public void testTimezonedTable4() throws Exception { + // Table - timezone = GMT+9 + // Client - GMT+9 (SET TIME ZONE 'GMT+9';) + + try { + executeDDL("datetime_table_timezoned_ddl.sql", "timezoned", new String[]{"timezoned4"}); + ResultSet res = executeQuery(); + assertResultSet(res, "testTimezonedTable3.result"); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned4"); + } + } + + @Test + public void testTimezonedTable5() throws Exception { + // Table - timezone = GMT+9 (by a specified system timezone) + // TajoClient uses JVM default timezone (GMT+9) + + try { + testingCluster.getConfiguration().setSystemTimezone(TimeZone.getTimeZone("GMT+9")); + + executeDDL("datetime_table_ddl.sql", "timezoned", new String[]{"timezoned5"}); + ResultSet res = executeQuery(); + assertResultSet(res, "testTimezonedTable3.result"); + cleanupQuery(res); + } finally { + executeString("DROP TABLE IF EXISTS timezoned5"); + + // restore the config + testingCluster.getConfiguration().setSystemTimezone(TimeZone.getTimeZone("GMT")); + } + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/dataset/TestJsonWithTimezone/timezoned/TestJsonWithTimezone.json ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/dataset/TestJsonWithTimezone/timezoned/TestJsonWithTimezone.json b/tajo-core-tests/src/test/resources/dataset/TestJsonWithTimezone/timezoned/TestJsonWithTimezone.json new file mode 100644 index 0000000..d621c6e --- /dev/null +++ b/tajo-core-tests/src/test/resources/dataset/TestJsonWithTimezone/timezoned/TestJsonWithTimezone.json @@ -0,0 +1,3 @@ +{"t_timestamp": "1980-4-1 01:50:30.010", "t_time": "01:50:30.010", "t_date": "1980-04-01"} +{"t_timestamp": "80/4/1 1:50:30 AM", "t_time": "1:50:30 AM", "t_date": "80/4/1"} +{"t_timestamp": "1980 April 1 1:50:30", "t_time": "1:50:30", "t_date": "1980-04-01"} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl b/tajo-core-tests/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl index 38e8bd9..74b2e1b 100644 --- a/tajo-core-tests/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl +++ b/tajo-core-tests/src/test/resources/dataset/TestSelectQuery/timezoned/table1.tbl @@ -1,3 +1,3 @@ -1980-4-1 01:50:30.010|1980-04-01 -80/4/1 1:50:30 AM|80/4/1 -1980 April 1 1:50:30|1980-04-01 \ No newline at end of file +1980-4-1 01:50:30.010|01:50:30.010|1980-04-01 +80/4/1 1:50:30 AM|1:50:30 AM|80/4/1 +1980 April 1 1:50:30|1:50:30|1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/datetime_table_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/datetime_table_ddl.sql b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/datetime_table_ddl.sql new file mode 100644 index 0000000..6cb7fa3 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/datetime_table_ddl.sql @@ -0,0 +1,5 @@ +CREATE EXTERNAL TABLE ${0} ( + t_timestamp TIMESTAMP, + t_time TIME, + t_date DATE +) USING JSON LOCATION ${table.path} http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/datetime_table_timezoned_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/datetime_table_timezoned_ddl.sql b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/datetime_table_timezoned_ddl.sql new file mode 100644 index 0000000..cc8d482 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/datetime_table_timezoned_ddl.sql @@ -0,0 +1,5 @@ +CREATE EXTERNAL TABLE ${0} ( + t_timestamp TIMESTAMP, + t_time TIME, + t_date DATE +) USING JSON WITH ('timezone' = 'GMT+9') LOCATION ${table.path} http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable1.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable1.sql b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable1.sql new file mode 100644 index 0000000..38c9e90 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable1.sql @@ -0,0 +1 @@ +SELECT * FROM timezoned1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable2.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable2.sql b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable2.sql new file mode 100644 index 0000000..1fd9e36 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable2.sql @@ -0,0 +1,2 @@ +SET TIME ZONE 'GMT'; +SELECT * FROM timezoned2; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable3.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable3.sql b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable3.sql new file mode 100644 index 0000000..32b9c3a --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable3.sql @@ -0,0 +1 @@ +SELECT * FROM timezoned3; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable4.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable4.sql b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable4.sql new file mode 100644 index 0000000..acd096b --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable4.sql @@ -0,0 +1,2 @@ +SET TIME ZONE 'GMT+9'; +SELECT * FROM timezoned4; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable5.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable5.sql b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable5.sql new file mode 100644 index 0000000..13894ce --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestJsonWithTimezone/testTimezonedTable5.sql @@ -0,0 +1,2 @@ +SET SESSION TIMEZONE = 'GMT+9'; +SELECT * FROM timezoned5; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql b/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql index d2c97c5..c5fd5ea 100644 --- a/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql +++ b/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_ddl.sql @@ -1,4 +1,5 @@ CREATE EXTERNAL TABLE ${0} ( t_timestamp TIMESTAMP, - t_date DATE + t_time TIME, + t_date DATE ) USING TEXT LOCATION ${table.path} http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql b/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql index 131e619..9c5d30d 100644 --- a/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql +++ b/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl.sql @@ -1,4 +1,5 @@ CREATE EXTERNAL TABLE ${0} ( t_timestamp TIMESTAMP, - t_date DATE + t_time TIME, + t_date DATE ) USING TEXT WITH ('timezone' = 'GMT+9') LOCATION ${table.path} http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl2.sql ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl2.sql b/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl2.sql index c02a653..1ebc6ba 100644 --- a/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl2.sql +++ b/tajo-core-tests/src/test/resources/queries/TestSelectQuery/datetime_table_timezoned_ddl2.sql @@ -1,4 +1,5 @@ CREATE TABLE ${0} ( t_timestamp TIMESTAMP, - t_date DATE + t_time TIME, + t_date DATE ) USING TEXT WITH ('timezone' = 'GMT+9') http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable1.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable1.result b/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable1.result new file mode 100644 index 0000000..825f31a --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable1.result @@ -0,0 +1,5 @@ +t_timestamp,t_time,t_date +------------------------------- +1980-04-01 01:50:30.01,01:50:30.01,1980-04-01 +1980-04-01 01:50:30,01:50:30,1980-04-01 +1980-04-01 01:50:30,01:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable2.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable2.result b/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable2.result new file mode 100644 index 0000000..8b13e55 --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable2.result @@ -0,0 +1,5 @@ +t_timestamp,t_time,t_date +------------------------------- +1980-03-31 16:50:30.01,16:50:30.01,1980-04-01 +1980-03-31 16:50:30,16:50:30,1980-04-01 +1980-03-31 16:50:30,16:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable3.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable3.result b/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable3.result new file mode 100644 index 0000000..825f31a --- /dev/null +++ b/tajo-core-tests/src/test/resources/results/TestJsonWithTimezone/testTimezonedTable3.result @@ -0,0 +1,5 @@ +t_timestamp,t_time,t_date +------------------------------- +1980-04-01 01:50:30.01,01:50:30.01,1980-04-01 +1980-04-01 01:50:30,01:50:30,1980-04-01 +1980-04-01 01:50:30,01:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result b/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result index 39f593b..825f31a 100644 --- a/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result +++ b/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable1.result @@ -1,5 +1,5 @@ -t_timestamp,t_date +t_timestamp,t_time,t_date ------------------------------- -1980-04-01 01:50:30.01,1980-04-01 -1980-04-01 01:50:30,1980-04-01 -1980-04-01 01:50:30,1980-04-01 \ No newline at end of file +1980-04-01 01:50:30.01,01:50:30.01,1980-04-01 +1980-04-01 01:50:30,01:50:30,1980-04-01 +1980-04-01 01:50:30,01:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result b/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result index 916f4be..8b13e55 100644 --- a/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result +++ b/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable2.result @@ -1,5 +1,5 @@ -t_timestamp,t_date +t_timestamp,t_time,t_date ------------------------------- -1980-03-31 16:50:30.01,1980-04-01 -1980-03-31 16:50:30,1980-04-01 -1980-03-31 16:50:30,1980-04-01 \ No newline at end of file +1980-03-31 16:50:30.01,16:50:30.01,1980-04-01 +1980-03-31 16:50:30,16:50:30,1980-04-01 +1980-03-31 16:50:30,16:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result ---------------------------------------------------------------------- diff --git a/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result b/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result index 39f593b..825f31a 100644 --- a/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result +++ b/tajo-core-tests/src/test/resources/results/TestSelectQuery/testTimezonedTable3.result @@ -1,5 +1,5 @@ -t_timestamp,t_date +t_timestamp,t_time,t_date ------------------------------- -1980-04-01 01:50:30.01,1980-04-01 -1980-04-01 01:50:30,1980-04-01 -1980-04-01 01:50:30,1980-04-01 \ No newline at end of file +1980-04-01 01:50:30.01,01:50:30.01,1980-04-01 +1980-04-01 01:50:30,01:50:30,1980-04-01 +1980-04-01 01:50:30,01:50:30,1980-04-01 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java index 0ce4a9c..091fc8f 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java @@ -26,12 +26,14 @@ import net.minidev.json.JSONObject; import net.minidev.json.parser.JSONParser; import net.minidev.json.parser.ParseException; import org.apache.commons.net.util.Base64; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.*; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.text.TextLineDeserializer; import org.apache.tajo.storage.text.TextLineParsingError; @@ -39,6 +41,7 @@ import org.apache.tajo.storage.text.TextLineParsingError; import java.io.IOException; import java.nio.charset.CharsetDecoder; import java.util.Map; +import java.util.TimeZone; public class JsonLineDeserializer extends TextLineDeserializer { private JSONParser parser; @@ -48,11 +51,17 @@ public class JsonLineDeserializer extends TextLineDeserializer { private final String [] projectedPaths; private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8); + private final boolean hasTimezone; + private final TimeZone timezone; + public JsonLineDeserializer(Schema schema, TableMeta meta, Column [] projected) { super(schema, meta); projectedPaths = SchemaUtil.convertColumnsToPaths(Lists.newArrayList(projected), true); types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); + + hasTimezone = meta.containsOption(StorageConstants.TIMEZONE); + timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); } @Override @@ -151,7 +160,11 @@ public class JsonLineDeserializer extends TextLineDeserializer { case TIMESTAMP: String timestampStr = object.getAsString(fieldName); if (timestampStr != null) { - output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr)); + if (hasTimezone) { + output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr, timezone)); + } else { + output.put(fieldIndex, DatumFactory.createTimestamp(timestampStr)); + } } else { output.put(fieldIndex, NullDatum.get()); } @@ -159,7 +172,11 @@ public class JsonLineDeserializer extends TextLineDeserializer { case TIME: String timeStr = object.getAsString(fieldName); if (timeStr != null) { - output.put(fieldIndex, DatumFactory.createTime(timeStr)); + if (hasTimezone) { + output.put(fieldIndex, DatumFactory.createTime(timeStr, timezone)); + } else { + output.put(fieldIndex, DatumFactory.createTime(timeStr)); + } } else { output.put(fieldIndex, NullDatum.get()); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3e8f4a03/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java index f283714..b42e14a 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java @@ -21,32 +21,42 @@ package org.apache.tajo.storage.json; import net.minidev.json.JSONObject; import org.apache.commons.net.util.Base64; +import org.apache.tajo.TajoConstants; import org.apache.tajo.catalog.NestedPathUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SchemaUtil; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.datum.TimeDatum; +import org.apache.tajo.datum.TimestampDatum; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.text.TextLineSerializer; import java.io.IOException; import java.io.OutputStream; import java.util.Map; +import java.util.TimeZone; public class JsonLineSerializer extends TextLineSerializer { // Full Path -> Type private final Map<String, Type> types; private final String [] projectedPaths; + private final boolean hasTimezone; + private final TimeZone timezone; public JsonLineSerializer(Schema schema, TableMeta meta) { super(schema, meta); projectedPaths = SchemaUtil.convertColumnsToPaths(schema.getAllColumns(), true); types = SchemaUtil.buildTypeMap(schema.getAllColumns(), projectedPaths); + + hasTimezone = meta.containsOption(StorageConstants.TIMEZONE); + timezone = TimeZone.getTimeZone(meta.getOption(StorageConstants.TIMEZONE, TajoConstants.DEFAULT_SYSTEM_TIMEZONE)); } @Override @@ -92,15 +102,32 @@ public class JsonLineSerializer extends TextLineSerializer { json.put(fieldName, input.getFloat8(fieldIndex)); break; - case CHAR: + case TEXT: case VARCHAR: + json.put(fieldName, input.getText(fieldIndex)); + break; + + case CHAR: case INET4: - case TIMESTAMP: case DATE: - case TIME: case INTERVAL: - json.put(fieldName, input.getText(fieldIndex)); + json.put(fieldName, input.asDatum(fieldIndex).asChars()); + break; + + case TIMESTAMP: + if (hasTimezone) { + json.put(fieldName, TimestampDatum.asChars(input.getTimeDate(fieldIndex), timezone, false)); + } else { + json.put(fieldName, input.asDatum(fieldIndex).asChars()); + } + break; + case TIME: + if (hasTimezone) { + json.put(fieldName, TimeDatum.asChars(input.getTimeDate(fieldIndex), timezone, false)); + } else { + json.put(fieldName, input.asDatum(fieldIndex).asChars()); + } break; case BIT:
