This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b66efb8284 Move schema related tests to pinot-spi (#13929)
b66efb8284 is described below
commit b66efb82845ead3daded4c3111a63d6131f35f15
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Tue Sep 3 22:08:15 2024 -0700
Move schema related tests to pinot-spi (#13929)
---
.../pinot/common/data/DateTimeFormatSpecTest.java | 378 ---------------------
.../common/data/DateTimeGranularitySpecTest.java | 86 -----
.../apache/pinot/common/utils/SchemaUtilsTest.java | 58 ++++
.../pinot/spi/data/DateTimeFieldSpecUtilsTest.java | 188 ----------
.../pinot/spi/data/DateTimeFormatSpecTest.java | 343 +++++++++++++++++++
.../spi/data/DateTimeGranularitySpecTest.java | 58 ++++
.../org/apache/pinot/spi}/data/FieldSpecTest.java | 8 +-
.../org/apache/pinot/spi}/data/SchemaTest.java | 225 +++++++++---
8 files changed, 639 insertions(+), 705 deletions(-)
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/data/DateTimeFormatSpecTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/data/DateTimeFormatSpecTest.java
deleted file mode 100644
index dd25994ce4..0000000000
---
a/pinot-common/src/test/java/org/apache/pinot/common/data/DateTimeFormatSpecTest.java
+++ /dev/null
@@ -1,378 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.common.data;
-
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-import org.apache.pinot.spi.data.DateTimeFieldSpec;
-import org.apache.pinot.spi.data.DateTimeFieldSpec.TimeFormat;
-import org.apache.pinot.spi.data.DateTimeFormatSpec;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.ISODateTimeFormat;
-import org.testng.Assert;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-
-/**
- * Tests for DateTimeFormatSpec helper methods
- */
-public class DateTimeFormatSpecTest {
-
- // Test conversion of a dateTimeColumn value from a format to millis
- @Test(dataProvider = "testFromFormatToMillisDataProvider")
- public void testFromFormatToMillis(String format, String formattedValue,
long expectedTimeMs) {
- Assert.assertEquals(new
DateTimeFormatSpec(format).fromFormatToMillis(formattedValue), expectedTimeMs);
- }
-
- @DataProvider(name = "testFromFormatToMillisDataProvider")
- public Object[][] provideTestFromFormatToMillisData() {
-
- List<Object[]> entries = new ArrayList<>();
- entries.add(new Object[]{"1:HOURS:EPOCH", "416359", 1498892400000L});
- entries.add(new Object[]{"1:MILLISECONDS:EPOCH", "1498892400000",
1498892400000L});
- entries.add(new Object[]{"1:HOURS:EPOCH", "0", 0L});
- entries.add(new Object[]{"5:MINUTES:EPOCH", "4996308", 1498892400000L});
- entries.add(new Object[]{
- "1:MILLISECONDS:TIMESTAMP", "2017-07-01 00:00:00",
Timestamp.valueOf("2017-07-01 00:00:00").getTime()
- });
- entries.add(new Object[]{"1:MILLISECONDS:TIMESTAMP", "1498892400000",
1498892400000L});
- entries.add(new Object[]{
- "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", "20170701",
-
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().parseMillis("20170701")
- });
- entries.add(new Object[]{
- "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(America/Chicago)", "20170701",
DateTimeFormat.forPattern("yyyyMMdd")
-
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/Chicago"))).parseMillis("20170701")
- });
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH", "20170701 00",
- DateTimeFormat.forPattern("yyyyMMdd
HH").withZoneUTC().parseMillis("20170701 00")
- });
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH tz(GMT+0600)", "20170701 00",
DateTimeFormat.forPattern("yyyyMMdd HH")
-
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+0600"))).parseMillis("20170701
00")
- });
- entries.add(new Object[]{"1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH Z",
"20170701 00 -07:00", 1498892400000L});
- entries.add(new Object[]{"1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a",
"8/7/2017 12:45:50 AM", 1502066750000L});
- entries.add(new Object[]{"EPOCH|HOURS|1", "416359", 1498892400000L});
- entries.add(new Object[]{"EPOCH|HOURS", "416359", 1498892400000L});
- entries.add(new Object[]{"EPOCH|MILLISECONDS|1", "1498892400000",
1498892400000L});
- entries.add(new Object[]{"EPOCH|MILLISECONDS", "1498892400000",
1498892400000L});
- entries.add(new Object[]{"EPOCH|HOURS|1", "0", 0L});
- entries.add(new Object[]{"EPOCH|HOURS", "0", 0L});
- entries.add(new Object[]{"EPOCH|MINUTES|5", "4996308", 1498892400000L});
- entries.add(new Object[]{
- "TIMESTAMP", "2017-07-01 00:00:00", Timestamp.valueOf("2017-07-01
00:00:00").getTime()
- });
- entries.add(new Object[]{"TIMESTAMP", "1498892400000", 1498892400000L});
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT", "2017-07-01",
-
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().parseMillis("20170701")
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT", "2017-07-01T12:45:50",
-
DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZoneUTC().parseMillis("2017-07-01T12:45:50")
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT", "2017",
- DateTimeFormat.forPattern("yyyy").withZoneUTC().parseMillis("2017")
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd", "20170701",
-
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().parseMillis("20170701")
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd|America/Chicago", "20170701",
DateTimeFormat.forPattern("yyyyMMdd")
-
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/Chicago"))).parseMillis("20170701")
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd HH", "20170701 00",
- DateTimeFormat.forPattern("yyyyMMdd
HH").withZoneUTC().parseMillis("20170701 00")
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd HH|GMT+0600", "20170701 00",
DateTimeFormat.forPattern("yyyyMMdd HH")
-
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+0600"))).parseMillis("20170701
00")
- });
- entries.add(new Object[]{"SIMPLE_DATE_FORMAT|yyyyMMdd HH Z", "20170701 00
-07:00", 1498892400000L});
- entries.add(new Object[]{"SIMPLE_DATE_FORMAT|M/d/yyyy h:mm:ss a",
"8/7/2017 12:45:50 AM", 1502066750000L});
-
- return entries.toArray(new Object[entries.size()][]);
- }
-
- // Test the conversion of a millis value to date time column value in a
format
- @Test(dataProvider = "testFromMillisToFormatDataProvider")
- public void testFromMillisToFormat(String format, long timeMs, String
expectedFormattedValue) {
- Assert.assertEquals(new
DateTimeFormatSpec(format).fromMillisToFormat(timeMs), expectedFormattedValue);
- }
-
- @DataProvider(name = "testFromMillisToFormatDataProvider")
- public Object[][] provideTestFromMillisToFormatData() {
-
- List<Object[]> entries = new ArrayList<>();
- entries.add(new Object[]{"1:HOURS:EPOCH", 1498892400000L, "416359"});
- entries.add(new Object[]{"1:MILLISECONDS:EPOCH", 1498892400000L,
"1498892400000"});
- entries.add(new Object[]{"1:HOURS:EPOCH", 0L, "0"});
- entries.add(new Object[]{"5:MINUTES:EPOCH", 1498892400000L, "4996308"});
- entries.add(new Object[]{
- "1:MILLISECONDS:TIMESTAMP", Timestamp.valueOf("2017-07-01
00:00:00").getTime(), "2017-07-01 00:00:00.0"
- });
- entries.add(new Object[]{
- "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", 1498892400000L,
-
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().print(1498892400000L)
- });
- entries.add(new Object[]{
- "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(America/New_York)",
1498892400000L, DateTimeFormat.forPattern("yyyyMMdd")
-
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/New_York"))).print(1498892400000L)
- });
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH", 1498892400000L,
- DateTimeFormat.forPattern("yyyyMMdd
HH").withZoneUTC().print(1498892400000L)
- });
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH tz(IST)", 1498892400000L,
- DateTimeFormat.forPattern("yyyyMMdd
HH").withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))).print(
- 1498892400000L)
- });
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH Z", 1498892400000L,
- DateTimeFormat.forPattern("yyyyMMdd HH
Z").withZoneUTC().print(1498892400000L)
- });
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH Z tz(GMT+0500)",
1498892400000L,
- DateTimeFormat.forPattern("yyyyMMdd HH Z")
-
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+0500"))).print(1498892400000L)
- });
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a", 1498892400000L,
- DateTimeFormat.forPattern("M/d/yyyy h:mm:ss
a").withZoneUTC().withLocale(Locale.ENGLISH).print(1498892400000L)
- });
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h a", 1502066750000L,
- DateTimeFormat.forPattern("M/d/yyyy h
a").withZoneUTC().withLocale(Locale.ENGLISH).print(1502066750000L)
- });
- entries.add(new Object[]{"EPOCH|HOURS|1", 1498892400000L, "416359"});
- entries.add(new Object[]{"EPOCH|MILLISECONDS|1", 1498892400000L,
"1498892400000"});
- entries.add(new Object[]{"EPOCH|HOURS|1", 0L, "0"});
- entries.add(new Object[]{"EPOCH|MINUTES|5", 1498892400000L, "4996308"});
- entries.add(new Object[]{
- "TIMESTAMP", Timestamp.valueOf("2017-07-01 00:00:00").getTime(),
"2017-07-01 00:00:00.0"
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd", 1498892400000L,
-
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().print(1498892400000L)
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd|America/New_York", 1498892400000L,
DateTimeFormat.forPattern("yyyyMMdd")
-
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/New_York"))).print(1498892400000L)
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd HH", 1498892400000L,
- DateTimeFormat.forPattern("yyyyMMdd
HH").withZoneUTC().print(1498892400000L)
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd HH|IST", 1498892400000L,
- DateTimeFormat.forPattern("yyyyMMdd
HH").withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))).print(
- 1498892400000L)
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd HH Z", 1498892400000L,
- DateTimeFormat.forPattern("yyyyMMdd HH
Z").withZoneUTC().print(1498892400000L)
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd HH Z|GMT+0500", 1498892400000L,
- DateTimeFormat.forPattern("yyyyMMdd HH Z")
-
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+0500"))).print(1498892400000L)
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|M/d/yyyy h:mm:ss a", 1498892400000L,
- DateTimeFormat.forPattern("M/d/yyyy h:mm:ss
a").withZoneUTC().withLocale(Locale.ENGLISH).print(1498892400000L)
- });
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|M/d/yyyy h a", 1502066750000L,
- DateTimeFormat.forPattern("M/d/yyyy h
a").withZoneUTC().withLocale(Locale.ENGLISH).print(1502066750000L)
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT", 1502066750000L,
-
ISODateTimeFormat.dateTimeNoMillis().withZoneUTC().withLocale(Locale.ENGLISH).print(1502066750000L)
- });
- return entries.toArray(new Object[entries.size()][]);
- }
-
- // Test fetching components of a format form a given format
- @Test(dataProvider = "testGetFromFormatDataProvider")
- public void testGetFromFormat(String format, int
columnSizeFromFormatExpected, TimeUnit columnUnitFromFormatExpected,
- TimeFormat timeFormatFromFormatExpected, String
sdfPatternFromFormatExpected,
- DateTimeZone dateTimeZoneFromFormatExpected) {
-
- DateTimeFormatSpec dateTimeFormatSpec = new DateTimeFormatSpec(format);
-
- int columnSizeFromFormat = dateTimeFormatSpec.getColumnSize();
- Assert.assertEquals(columnSizeFromFormat, columnSizeFromFormatExpected);
-
- TimeUnit columnUnitFromFormat = dateTimeFormatSpec.getColumnUnit();
- Assert.assertEquals(columnUnitFromFormat, columnUnitFromFormatExpected);
-
- TimeFormat timeFormatFromFormat = dateTimeFormatSpec.getTimeFormat();
- Assert.assertEquals(timeFormatFromFormat, timeFormatFromFormatExpected);
-
- String sdfPatternFromFormat = null;
- DateTimeZone dateTimeZoneFromFormat = DateTimeZone.UTC;
- try {
- sdfPatternFromFormat = dateTimeFormatSpec.getSDFPattern();
- dateTimeZoneFromFormat = dateTimeFormatSpec.getDateTimezone();
- } catch (Exception e) {
- // No sdf pattern
- }
- Assert.assertEquals(sdfPatternFromFormat, sdfPatternFromFormatExpected);
- Assert.assertEquals(dateTimeZoneFromFormat,
dateTimeZoneFromFormatExpected);
- }
-
- @DataProvider(name = "testGetFromFormatDataProvider")
- public Object[][] provideTestGetFromFormatData() {
-
- List<Object[]> entries = new ArrayList<>();
-
- entries.add(
- new Object[]{"1:MILLISECONDS:TIMESTAMP", 1, TimeUnit.MILLISECONDS,
TimeFormat.TIMESTAMP, null,
- DateTimeZone.UTC});
-
- entries.add(
- new Object[]{"1:HOURS:EPOCH", 1, TimeUnit.HOURS,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC});
-
- entries.add(new Object[]{
- "5:MINUTES:EPOCH", 5, TimeUnit.MINUTES,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
- "yyyyMMdd", DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(IST)", 1, TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
DateTimeZone.forTimeZone(
- TimeZone.getTimeZone("IST"))
- });
-
- entries.add(new Object[]{
- "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(IST)", 1,
TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
- DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
- });
-
- entries.add(new Object[]{
- "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz ( IST ) ", 1,
TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
- DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
- });
-
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH", 1, TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd HH",
DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH tz(dummy)", 1,
TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd HH",
DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a", 1,
TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "M/d/yyyy h:mm:ss a",
DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a tz(Asia/Tokyo)", 1,
TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "M/d/yyyy h:mm:ss a",
- DateTimeZone.forTimeZone(TimeZone.getTimeZone("Asia/Tokyo"))
- });
-
- //test new format
- entries.add(
- new Object[]{"TIMESTAMP", 1, TimeUnit.MILLISECONDS,
TimeFormat.TIMESTAMP, null,
- DateTimeZone.UTC});
-
- entries.add(
- new Object[]{"EPOCH", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC});
-
- entries.add(
- new Object[]{"EPOCH|HOURS|1", 1, TimeUnit.HOURS,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC});
-
- entries.add(new Object[]{
- "EPOCH|MINUTES|5", 5, TimeUnit.MINUTES,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
- null, DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
- "yyyyMMdd", DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd|IST", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
- "yyyyMMdd", DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd|IST", 1, TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
- DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd|IST", 1, TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
- DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd HH", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
- "yyyyMMdd HH", DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|yyyyMMdd HH|dummy", 1, TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd HH",
DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|M/d/yyyy h:mm:ss a", 1, TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "M/d/yyyy h:mm:ss a",
DateTimeZone.UTC
- });
-
- entries.add(new Object[]{
- "SIMPLE_DATE_FORMAT|M/d/yyyy h:mm:ss a|Asia/Tokyo", 1,
TimeUnit.MILLISECONDS,
- DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "M/d/yyyy h:mm:ss a",
- DateTimeZone.forTimeZone(TimeZone.getTimeZone("Asia/Tokyo"))
- });
- return entries.toArray(new Object[entries.size()][]);
- }
-}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/data/DateTimeGranularitySpecTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/data/DateTimeGranularitySpecTest.java
deleted file mode 100644
index d6a2f62223..0000000000
---
a/pinot-common/src/test/java/org/apache/pinot/common/data/DateTimeGranularitySpecTest.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.common.data;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import org.apache.pinot.spi.data.DateTimeGranularitySpec;
-import org.testng.Assert;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-
-public class DateTimeGranularitySpecTest {
-
- // Test construct granularity from components
- @Test(dataProvider = "testConstructGranularityDataProvider")
- public void testConstructGranularity(int size, TimeUnit unit,
DateTimeGranularitySpec granularityExpected) {
- DateTimeGranularitySpec granularityActual = null;
- try {
- granularityActual = new DateTimeGranularitySpec(size, unit);
- } catch (Exception e) {
- // invalid arguments
- }
- Assert.assertEquals(granularityActual, granularityExpected);
- }
-
- @DataProvider(name = "testConstructGranularityDataProvider")
- public Object[][] provideTestConstructGranularityData() {
-
- List<Object[]> entries = new ArrayList<>();
-
- entries.add(new Object[]{1, TimeUnit.HOURS, new
DateTimeGranularitySpec("1:HOURS")});
- entries.add(new Object[]{5, TimeUnit.MINUTES, new
DateTimeGranularitySpec("5:MINUTES")});
- entries.add(new Object[]{0, TimeUnit.HOURS, null});
- entries.add(new Object[]{-1, TimeUnit.HOURS, null});
- entries.add(new Object[]{1, null, null});
-
- return entries.toArray(new Object[entries.size()][]);
- }
-
- // Test granularity to millis
- @Test(dataProvider = "testGranularityToMillisDataProvider")
- public void testGranularityToMillis(String granularity, Long millisExpected)
{
- Long millisActual = null;
- DateTimeGranularitySpec granularitySpec = null;
- try {
- granularitySpec = new DateTimeGranularitySpec(granularity);
- millisActual = granularitySpec.granularityToMillis();
- } catch (Exception e) {
- // invalid arguments
- }
- Assert.assertEquals(millisActual, millisExpected);
- }
-
- @DataProvider(name = "testGranularityToMillisDataProvider")
- public Object[][] provideTestGranularityToMillisData() {
-
- List<Object[]> entries = new ArrayList<>();
-
- entries.add(new Object[]{"1:HOURS", 3600000L});
- entries.add(new Object[]{"1:MILLISECONDS", 1L});
- entries.add(new Object[]{"15:MINUTES", 900000L});
- entries.add(new Object[]{"0:HOURS", null});
- entries.add(new Object[]{null, null});
- entries.add(new Object[]{"1:DUMMY", null});
-
- return entries.toArray(new Object[entries.size()][]);
- }
-}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/SchemaUtilsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/SchemaUtilsTest.java
new file mode 100644
index 0000000000..90a554b26c
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/SchemaUtilsTest.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.utils;
+
+import java.io.File;
+import java.net.URL;
+import org.apache.pinot.spi.data.Schema;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertNotNull;
+
+
+public class SchemaUtilsTest {
+
+ @Test
+ public void testSchemaSerDe()
+ throws Exception {
+ URL resourceUrl =
getClass().getClassLoader().getResource("schemaTest.schema");
+ assertNotNull(resourceUrl);
+ Schema schema = Schema.fromFile(new File(resourceUrl.getFile()));
+
+ Schema schemaToCompare = Schema.fromString(schema.toPrettyJsonString());
+ assertEquals(schemaToCompare, schema);
+ assertEquals(schemaToCompare.hashCode(), schema.hashCode());
+
+ schemaToCompare = Schema.fromString(schema.toSingleLineJsonString());
+ assertEquals(schemaToCompare, schema);
+ assertEquals(schemaToCompare.hashCode(), schema.hashCode());
+
+ schemaToCompare = SchemaUtils.fromZNRecord(SchemaUtils.toZNRecord(schema));
+ assertEquals(schemaToCompare, schema);
+ assertEquals(schemaToCompare.hashCode(), schema.hashCode());
+
+ // When setting new fields, schema string should be updated
+ String jsonSchema = schemaToCompare.toSingleLineJsonString();
+ schemaToCompare.setSchemaName("newSchema");
+ String jsonSchemaToCompare = schemaToCompare.toSingleLineJsonString();
+ assertNotEquals(jsonSchemaToCompare, jsonSchema);
+ }
+}
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeFieldSpecUtilsTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeFieldSpecUtilsTest.java
deleted file mode 100644
index 837cd1db9d..0000000000
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeFieldSpecUtilsTest.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.spi.data;
-
-import java.util.concurrent.TimeUnit;
-import org.apache.pinot.spi.data.FieldSpec.DataType;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-
-/**
- * Tests the conversion of a {@link TimeFieldSpec} to an equivalent {@link
DateTimeFieldSpec}
- */
-public class DateTimeFieldSpecUtilsTest {
-
- @Test
- public void testConversionFromTimeToDateTimeSpec() {
- TimeFieldSpec timeFieldSpec;
- DateTimeFieldSpec expectedDateTimeFieldSpec;
- DateTimeFieldSpec actualDateTimeFieldSpec;
-
- /* 1] only incoming */
-
- // incoming epoch millis
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
TimeUnit.MILLISECONDS, "incoming"));
- expectedDateTimeFieldSpec =
- new DateTimeFieldSpec("incoming", DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // incoming epoch hours
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.INT,
TimeUnit.HOURS, "incoming"));
- expectedDateTimeFieldSpec = new DateTimeFieldSpec("incoming",
DataType.INT, "1:HOURS:EPOCH", "1:HOURS");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // Simple date format
- timeFieldSpec = new TimeFieldSpec(
- new TimeGranularitySpec(DataType.INT, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyyMMdd", "incoming"));
- expectedDateTimeFieldSpec =
- new DateTimeFieldSpec("incoming", DataType.INT,
"1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", "1:DAYS");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // simple date format STRING
- timeFieldSpec = new TimeFieldSpec(
- new TimeGranularitySpec(DataType.STRING, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyy-MM-dd hh-mm-ss", "incoming"));
- expectedDateTimeFieldSpec =
- new DateTimeFieldSpec("incoming", DataType.STRING,
"1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd hh-mm-ss", "1:DAYS");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // time unit size
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
5, TimeUnit.MINUTES, "incoming"));
- expectedDateTimeFieldSpec = new DateTimeFieldSpec("incoming",
DataType.LONG, "5:MINUTES:EPOCH", "5:MINUTES");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // transform function
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.INT,
TimeUnit.HOURS, "incoming"));
- timeFieldSpec.setTransformFunction("toEpochHours(timestamp)");
- expectedDateTimeFieldSpec =
- new DateTimeFieldSpec("incoming", DataType.INT, "1:HOURS:EPOCH",
"1:HOURS", null, "toEpochHours(timestamp)");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- /* 2] incoming + outgoing */
-
- // same incoming and outgoing
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
TimeUnit.HOURS, "time"),
- new TimeGranularitySpec(DataType.LONG, TimeUnit.HOURS, "time"));
- expectedDateTimeFieldSpec = new DateTimeFieldSpec("time", DataType.LONG,
"1:HOURS:EPOCH", "1:HOURS");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // same incoming and outgoing - simple date format
- timeFieldSpec =
- new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
TimeUnit.DAYS, "SIMPLE_DATE_FORMAT:yyyyMMdd", "time"),
- new TimeGranularitySpec(DataType.LONG, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyyMMdd", "time"));
- expectedDateTimeFieldSpec =
- new DateTimeFieldSpec("time", DataType.LONG,
"1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", "1:DAYS");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // millis to hours
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
TimeUnit.MILLISECONDS, "incoming"),
- new TimeGranularitySpec(DataType.LONG, TimeUnit.HOURS, "outgoing"));
- expectedDateTimeFieldSpec =
- new DateTimeFieldSpec("outgoing", DataType.LONG, "1:HOURS:EPOCH",
"1:HOURS", null, "toEpochHours(incoming)");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // millis to bucketed minutes
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
TimeUnit.MILLISECONDS, "incoming"),
- new TimeGranularitySpec(DataType.LONG, 10, TimeUnit.MINUTES,
"outgoing"));
- expectedDateTimeFieldSpec = new DateTimeFieldSpec("outgoing",
DataType.LONG, "10:MINUTES:EPOCH", "10:MINUTES", null,
- "toEpochMinutesBucket(incoming, 10)");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // days to millis
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.INT,
TimeUnit.DAYS, "incoming"),
- new TimeGranularitySpec(DataType.LONG, TimeUnit.MILLISECONDS,
"outgoing"));
- expectedDateTimeFieldSpec =
- new DateTimeFieldSpec("outgoing", DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS", null,
- "fromEpochDays(incoming)");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // bucketed minutes to millis
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
5, TimeUnit.MINUTES, "incoming"),
- new TimeGranularitySpec(DataType.LONG, TimeUnit.MILLISECONDS,
"outgoing"));
- expectedDateTimeFieldSpec =
- new DateTimeFieldSpec("outgoing", DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS", null,
- "fromEpochMinutesBucket(incoming, 5)");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // hours to days
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.INT,
TimeUnit.HOURS, "incoming"),
- new TimeGranularitySpec(DataType.INT, TimeUnit.DAYS, "outgoing"));
- expectedDateTimeFieldSpec = new DateTimeFieldSpec("outgoing",
DataType.INT, "1:DAYS:EPOCH", "1:DAYS", null,
- "toEpochDays(fromEpochHours(incoming))");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // minutes to hours
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
TimeUnit.MINUTES, "incoming"),
- new TimeGranularitySpec(DataType.LONG, TimeUnit.HOURS, "outgoing"));
- expectedDateTimeFieldSpec = new DateTimeFieldSpec("outgoing",
DataType.LONG, "1:HOURS:EPOCH", "1:HOURS", null,
- "toEpochHours(fromEpochMinutes(incoming))");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // bucketed minutes to days
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
10, TimeUnit.MINUTES, "incoming"),
- new TimeGranularitySpec(DataType.LONG, TimeUnit.DAYS, "outgoing"));
- expectedDateTimeFieldSpec = new DateTimeFieldSpec("outgoing",
DataType.LONG, "1:DAYS:EPOCH", "1:DAYS", null,
- "toEpochDays(fromEpochMinutesBucket(incoming, 10))");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // seconds to bucketed minutes
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
TimeUnit.SECONDS, "incoming"),
- new TimeGranularitySpec(DataType.LONG, 5, TimeUnit.MINUTES,
"outgoing"));
- expectedDateTimeFieldSpec = new DateTimeFieldSpec("outgoing",
DataType.LONG, "5:MINUTES:EPOCH", "5:MINUTES", null,
- "toEpochMinutesBucket(fromEpochSeconds(incoming), 5)");
- actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
-
- // simple date format to millis
- timeFieldSpec = new TimeFieldSpec(
- new TimeGranularitySpec(DataType.LONG, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyyMMdd", "incoming"),
- new TimeGranularitySpec(DataType.LONG, TimeUnit.MILLISECONDS,
"outgoing"));
- try {
- Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.fail();
- } catch (Exception e) {
- // expected
- }
-
- // hours to simple date format
- timeFieldSpec = new TimeFieldSpec(new TimeGranularitySpec(DataType.LONG,
TimeUnit.HOURS, "incoming"),
- new TimeGranularitySpec(DataType.INT, TimeUnit.HOURS,
"SIMPLE_DATE_FORMAT:yyyyMMddhh", "outgoing"));
- try {
- Schema.convertToDateTimeFieldSpec(timeFieldSpec);
- Assert.fail();
- } catch (Exception e) {
- // expected
- }
- }
-}
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeFormatSpecTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeFormatSpecTest.java
index 9bf1c122d4..898e182fb3 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeFormatSpecTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeFormatSpecTest.java
@@ -18,9 +18,16 @@
*/
package org.apache.pinot.spi.data;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.ISODateTimeFormat;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@@ -30,6 +37,342 @@ import static org.testng.Assert.assertThrows;
public class DateTimeFormatSpecTest {
+ // Test conversion of a dateTimeColumn value from a format to millis
+ @Test(dataProvider = "testFromFormatToMillisDataProvider")
+ public void testFromFormatToMillis(String format, String formattedValue,
long expectedTimeMs) {
+ assertEquals(new
DateTimeFormatSpec(format).fromFormatToMillis(formattedValue), expectedTimeMs);
+ }
+
+ @DataProvider(name = "testFromFormatToMillisDataProvider")
+ public Object[][] provideTestFromFormatToMillisData() {
+
+ List<Object[]> entries = new ArrayList<>();
+ entries.add(new Object[]{"1:HOURS:EPOCH", "416359", 1498892400000L});
+ entries.add(new Object[]{"1:MILLISECONDS:EPOCH", "1498892400000",
1498892400000L});
+ entries.add(new Object[]{"1:HOURS:EPOCH", "0", 0L});
+ entries.add(new Object[]{"5:MINUTES:EPOCH", "4996308", 1498892400000L});
+ entries.add(new Object[]{
+ "1:MILLISECONDS:TIMESTAMP", "2017-07-01 00:00:00",
Timestamp.valueOf("2017-07-01 00:00:00").getTime()
+ });
+ entries.add(new Object[]{"1:MILLISECONDS:TIMESTAMP", "1498892400000",
1498892400000L});
+ entries.add(new Object[]{
+ "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", "20170701",
+
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().parseMillis("20170701")
+ });
+ entries.add(new Object[]{
+ "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(America/Chicago)", "20170701",
DateTimeFormat.forPattern("yyyyMMdd")
+
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/Chicago"))).parseMillis("20170701")
+ });
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH", "20170701 00",
+ DateTimeFormat.forPattern("yyyyMMdd
HH").withZoneUTC().parseMillis("20170701 00")
+ });
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH tz(GMT+0600)", "20170701 00",
DateTimeFormat.forPattern("yyyyMMdd HH")
+
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+0600"))).parseMillis("20170701
00")
+ });
+ entries.add(new Object[]{"1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH Z",
"20170701 00 -07:00", 1498892400000L});
+ entries.add(new Object[]{"1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a",
"8/7/2017 12:45:50 AM", 1502066750000L});
+ entries.add(new Object[]{"EPOCH|HOURS|1", "416359", 1498892400000L});
+ entries.add(new Object[]{"EPOCH|HOURS", "416359", 1498892400000L});
+ entries.add(new Object[]{"EPOCH|MILLISECONDS|1", "1498892400000",
1498892400000L});
+ entries.add(new Object[]{"EPOCH|MILLISECONDS", "1498892400000",
1498892400000L});
+ entries.add(new Object[]{"EPOCH|HOURS|1", "0", 0L});
+ entries.add(new Object[]{"EPOCH|HOURS", "0", 0L});
+ entries.add(new Object[]{"EPOCH|MINUTES|5", "4996308", 1498892400000L});
+ entries.add(new Object[]{
+ "TIMESTAMP", "2017-07-01 00:00:00", Timestamp.valueOf("2017-07-01
00:00:00").getTime()
+ });
+ entries.add(new Object[]{"TIMESTAMP", "1498892400000", 1498892400000L});
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT", "2017-07-01",
+
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().parseMillis("20170701")
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT", "2017-07-01T12:45:50",
+
DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZoneUTC().parseMillis("2017-07-01T12:45:50")
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT", "2017",
+ DateTimeFormat.forPattern("yyyy").withZoneUTC().parseMillis("2017")
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd", "20170701",
+
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().parseMillis("20170701")
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd|America/Chicago", "20170701",
DateTimeFormat.forPattern("yyyyMMdd")
+
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/Chicago"))).parseMillis("20170701")
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd HH", "20170701 00",
+ DateTimeFormat.forPattern("yyyyMMdd
HH").withZoneUTC().parseMillis("20170701 00")
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd HH|GMT+0600", "20170701 00",
DateTimeFormat.forPattern("yyyyMMdd HH")
+
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+0600"))).parseMillis("20170701
00")
+ });
+ entries.add(new Object[]{"SIMPLE_DATE_FORMAT|yyyyMMdd HH Z", "20170701 00
-07:00", 1498892400000L});
+ entries.add(new Object[]{"SIMPLE_DATE_FORMAT|M/d/yyyy h:mm:ss a",
"8/7/2017 12:45:50 AM", 1502066750000L});
+
+ return entries.toArray(new Object[entries.size()][]);
+ }
+
+ // Test the conversion of a millis value to date time column value in a
format
+ @Test(dataProvider = "testFromMillisToFormatDataProvider")
+ public void testFromMillisToFormat(String format, long timeMs, String
expectedFormattedValue) {
+ assertEquals(new DateTimeFormatSpec(format).fromMillisToFormat(timeMs),
expectedFormattedValue);
+ }
+
+ @DataProvider(name = "testFromMillisToFormatDataProvider")
+ public Object[][] provideTestFromMillisToFormatData() {
+
+ List<Object[]> entries = new ArrayList<>();
+ entries.add(new Object[]{"1:HOURS:EPOCH", 1498892400000L, "416359"});
+ entries.add(new Object[]{"1:MILLISECONDS:EPOCH", 1498892400000L,
"1498892400000"});
+ entries.add(new Object[]{"1:HOURS:EPOCH", 0L, "0"});
+ entries.add(new Object[]{"5:MINUTES:EPOCH", 1498892400000L, "4996308"});
+ entries.add(new Object[]{
+ "1:MILLISECONDS:TIMESTAMP", Timestamp.valueOf("2017-07-01
00:00:00").getTime(), "2017-07-01 00:00:00.0"
+ });
+ entries.add(new Object[]{
+ "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", 1498892400000L,
+
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(America/New_York)",
1498892400000L, DateTimeFormat.forPattern("yyyyMMdd")
+
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/New_York"))).print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH", 1498892400000L,
+ DateTimeFormat.forPattern("yyyyMMdd
HH").withZoneUTC().print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH tz(IST)", 1498892400000L,
+ DateTimeFormat.forPattern("yyyyMMdd
HH").withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))).print(
+ 1498892400000L)
+ });
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH Z", 1498892400000L,
+ DateTimeFormat.forPattern("yyyyMMdd HH
Z").withZoneUTC().print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH Z tz(GMT+0500)",
1498892400000L,
+ DateTimeFormat.forPattern("yyyyMMdd HH Z")
+
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+0500"))).print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a", 1498892400000L,
+ DateTimeFormat.forPattern("M/d/yyyy h:mm:ss
a").withZoneUTC().withLocale(Locale.ENGLISH).print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h a", 1502066750000L,
+ DateTimeFormat.forPattern("M/d/yyyy h
a").withZoneUTC().withLocale(Locale.ENGLISH).print(1502066750000L)
+ });
+ entries.add(new Object[]{"EPOCH|HOURS|1", 1498892400000L, "416359"});
+ entries.add(new Object[]{"EPOCH|MILLISECONDS|1", 1498892400000L,
"1498892400000"});
+ entries.add(new Object[]{"EPOCH|HOURS|1", 0L, "0"});
+ entries.add(new Object[]{"EPOCH|MINUTES|5", 1498892400000L, "4996308"});
+ entries.add(new Object[]{
+ "TIMESTAMP", Timestamp.valueOf("2017-07-01 00:00:00").getTime(),
"2017-07-01 00:00:00.0"
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd", 1498892400000L,
+
DateTimeFormat.forPattern("yyyyMMdd").withZoneUTC().print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd|America/New_York", 1498892400000L,
DateTimeFormat.forPattern("yyyyMMdd")
+
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("America/New_York"))).print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd HH", 1498892400000L,
+ DateTimeFormat.forPattern("yyyyMMdd
HH").withZoneUTC().print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd HH|IST", 1498892400000L,
+ DateTimeFormat.forPattern("yyyyMMdd
HH").withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))).print(
+ 1498892400000L)
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd HH Z", 1498892400000L,
+ DateTimeFormat.forPattern("yyyyMMdd HH
Z").withZoneUTC().print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd HH Z|GMT+0500", 1498892400000L,
+ DateTimeFormat.forPattern("yyyyMMdd HH Z")
+
.withZone(DateTimeZone.forTimeZone(TimeZone.getTimeZone("GMT+0500"))).print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|M/d/yyyy h:mm:ss a", 1498892400000L,
+ DateTimeFormat.forPattern("M/d/yyyy h:mm:ss
a").withZoneUTC().withLocale(Locale.ENGLISH).print(1498892400000L)
+ });
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|M/d/yyyy h a", 1502066750000L,
+ DateTimeFormat.forPattern("M/d/yyyy h
a").withZoneUTC().withLocale(Locale.ENGLISH).print(1502066750000L)
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT", 1502066750000L,
+
ISODateTimeFormat.dateTimeNoMillis().withZoneUTC().withLocale(Locale.ENGLISH).print(1502066750000L)
+ });
+ return entries.toArray(new Object[entries.size()][]);
+ }
+
+ // Test fetching components of a format form a given format
+ @Test(dataProvider = "testGetFromFormatDataProvider")
+ public void testGetFromFormat(String format, int
columnSizeFromFormatExpected, TimeUnit columnUnitFromFormatExpected,
+ DateTimeFieldSpec.TimeFormat timeFormatFromFormatExpected, String
sdfPatternFromFormatExpected,
+ DateTimeZone dateTimeZoneFromFormatExpected) {
+
+ DateTimeFormatSpec dateTimeFormatSpec = new DateTimeFormatSpec(format);
+
+ int columnSizeFromFormat = dateTimeFormatSpec.getColumnSize();
+ assertEquals(columnSizeFromFormat, columnSizeFromFormatExpected);
+
+ TimeUnit columnUnitFromFormat = dateTimeFormatSpec.getColumnUnit();
+ assertEquals(columnUnitFromFormat, columnUnitFromFormatExpected);
+
+ DateTimeFieldSpec.TimeFormat timeFormatFromFormat =
dateTimeFormatSpec.getTimeFormat();
+ assertEquals(timeFormatFromFormat, timeFormatFromFormatExpected);
+
+ String sdfPatternFromFormat = null;
+ DateTimeZone dateTimeZoneFromFormat = DateTimeZone.UTC;
+ try {
+ sdfPatternFromFormat = dateTimeFormatSpec.getSDFPattern();
+ dateTimeZoneFromFormat = dateTimeFormatSpec.getDateTimezone();
+ } catch (Exception e) {
+ // No sdf pattern
+ }
+ assertEquals(sdfPatternFromFormat, sdfPatternFromFormatExpected);
+ assertEquals(dateTimeZoneFromFormat, dateTimeZoneFromFormatExpected);
+ }
+
+ @DataProvider(name = "testGetFromFormatDataProvider")
+ public Object[][] provideTestGetFromFormatData() {
+
+ List<Object[]> entries = new ArrayList<>();
+
+ entries.add(
+ new Object[]{"1:MILLISECONDS:TIMESTAMP", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.TIMESTAMP, null,
+ DateTimeZone.UTC});
+
+ entries.add(
+ new Object[]{"1:HOURS:EPOCH", 1, TimeUnit.HOURS,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC});
+
+ entries.add(new Object[]{
+ "5:MINUTES:EPOCH", 5, TimeUnit.MINUTES,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
+ "yyyyMMdd", DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(IST)", 1, TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
DateTimeZone.forTimeZone(
+ TimeZone.getTimeZone("IST"))
+ });
+
+ entries.add(new Object[]{
+ "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz(IST)", 1,
TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
+ DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
+ });
+
+ entries.add(new Object[]{
+ "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd tz ( IST ) ", 1,
TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
+ DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
+ });
+
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH", 1, TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd HH",
DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:yyyyMMdd HH tz(dummy)", 1,
TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd HH",
DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a", 1,
TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "M/d/yyyy h:mm:ss a",
DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "1:HOURS:SIMPLE_DATE_FORMAT:M/d/yyyy h:mm:ss a tz(Asia/Tokyo)", 1,
TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "M/d/yyyy h:mm:ss a",
+ DateTimeZone.forTimeZone(TimeZone.getTimeZone("Asia/Tokyo"))
+ });
+
+ //test new format
+ entries.add(
+ new Object[]{"TIMESTAMP", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.TIMESTAMP, null,
+ DateTimeZone.UTC});
+
+ entries.add(
+ new Object[]{"EPOCH", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC});
+
+ entries.add(
+ new Object[]{"EPOCH|HOURS|1", 1, TimeUnit.HOURS,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC});
+
+ entries.add(new Object[]{
+ "EPOCH|MINUTES|5", 5, TimeUnit.MINUTES,
DateTimeFieldSpec.TimeFormat.EPOCH, null, DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
+ null, DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
+ "yyyyMMdd", DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd|IST", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
+ "yyyyMMdd", DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd|IST", 1, TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
+ DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd|IST", 1, TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd",
+ DateTimeZone.forTimeZone(TimeZone.getTimeZone("IST"))
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd HH", 1, TimeUnit.MILLISECONDS,
DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT,
+ "yyyyMMdd HH", DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|yyyyMMdd HH|dummy", 1, TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "yyyyMMdd HH",
DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|M/d/yyyy h:mm:ss a", 1, TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "M/d/yyyy h:mm:ss a",
DateTimeZone.UTC
+ });
+
+ entries.add(new Object[]{
+ "SIMPLE_DATE_FORMAT|M/d/yyyy h:mm:ss a|Asia/Tokyo", 1,
TimeUnit.MILLISECONDS,
+ DateTimeFieldSpec.TimeFormat.SIMPLE_DATE_FORMAT, "M/d/yyyy h:mm:ss a",
+ DateTimeZone.forTimeZone(TimeZone.getTimeZone("Asia/Tokyo"))
+ });
+ return entries.toArray(new Object[entries.size()][]);
+ }
+
@Test
public void testDateTimeFormatSpec() {
DateTimeFormatSpec dateTimeFormatSpec = new
DateTimeFormatSpec("5:DAYS:EPOCH");
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeGranularitySpecTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeGranularitySpecTest.java
index ed60c5d915..45cbc000a4 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeGranularitySpecTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/DateTimeGranularitySpecTest.java
@@ -19,7 +19,10 @@
package org.apache.pinot.spi.data;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.TimeUnit;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
@@ -28,6 +31,61 @@ import static org.testng.Assert.assertThrows;
public class DateTimeGranularitySpecTest {
+ // Test construct granularity from components
+ @Test(dataProvider = "testConstructGranularityDataProvider")
+ public void testConstructGranularity(int size, TimeUnit unit,
DateTimeGranularitySpec granularityExpected) {
+ DateTimeGranularitySpec granularityActual = null;
+ try {
+ granularityActual = new DateTimeGranularitySpec(size, unit);
+ } catch (Exception e) {
+ // invalid arguments
+ }
+ assertEquals(granularityActual, granularityExpected);
+ }
+
+ @DataProvider(name = "testConstructGranularityDataProvider")
+ public Object[][] provideTestConstructGranularityData() {
+
+ List<Object[]> entries = new ArrayList<>();
+
+ entries.add(new Object[]{1, TimeUnit.HOURS, new
DateTimeGranularitySpec("1:HOURS")});
+ entries.add(new Object[]{5, TimeUnit.MINUTES, new
DateTimeGranularitySpec("5:MINUTES")});
+ entries.add(new Object[]{0, TimeUnit.HOURS, null});
+ entries.add(new Object[]{-1, TimeUnit.HOURS, null});
+ entries.add(new Object[]{1, null, null});
+
+ return entries.toArray(new Object[entries.size()][]);
+ }
+
+ // Test granularity to millis
+ @Test(dataProvider = "testGranularityToMillisDataProvider")
+ public void testGranularityToMillis(String granularity, Long millisExpected)
{
+ Long millisActual = null;
+ DateTimeGranularitySpec granularitySpec = null;
+ try {
+ granularitySpec = new DateTimeGranularitySpec(granularity);
+ millisActual = granularitySpec.granularityToMillis();
+ } catch (Exception e) {
+ // invalid arguments
+ }
+ assertEquals(millisActual, millisExpected);
+ }
+
+ @DataProvider(name = "testGranularityToMillisDataProvider")
+ public Object[][] provideTestGranularityToMillisData() {
+
+ List<Object[]> entries = new ArrayList<>();
+
+ entries.add(new Object[]{"1:HOURS", 3600000L});
+ entries.add(new Object[]{"1:MILLISECONDS", 1L});
+ entries.add(new Object[]{"15:MINUTES", 900000L});
+ entries.add(new Object[]{"0:HOURS", null});
+ entries.add(new Object[]{null, null});
+ entries.add(new Object[]{"1:DUMMY", null});
+
+ return entries.toArray(new Object[entries.size()][]);
+ }
+
@Test
public void testDateTimeGranularitySpec() {
// Old format
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
similarity index 98%
rename from
pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
rename to pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
index 98f03b2ed9..10f7463990 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.common.data;
+package org.apache.pinot.spi.data;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
@@ -27,12 +27,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-import org.apache.pinot.spi.data.DateTimeFieldSpec;
-import org.apache.pinot.spi.data.DimensionFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.MetricFieldSpec;
-import org.apache.pinot.spi.data.TimeFieldSpec;
-import org.apache.pinot.spi.data.TimeGranularitySpec;
import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/data/SchemaTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaTest.java
similarity index 72%
rename from
pinot-common/src/test/java/org/apache/pinot/common/data/SchemaTest.java
rename to pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaTest.java
index e8fd128729..c4a2aee5be 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/data/SchemaTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaTest.java
@@ -16,23 +16,13 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pinot.common.data;
+package org.apache.pinot.spi.data;
import com.fasterxml.jackson.databind.JsonNode;
-import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
-import java.net.URL;
import java.sql.Timestamp;
import java.util.concurrent.TimeUnit;
-import org.apache.pinot.common.utils.SchemaUtils;
-import org.apache.pinot.spi.data.DateTimeFieldSpec;
-import org.apache.pinot.spi.data.DimensionFieldSpec;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.MetricFieldSpec;
-import org.apache.pinot.spi.data.Schema;
-import org.apache.pinot.spi.data.TimeFieldSpec;
-import org.apache.pinot.spi.data.TimeGranularitySpec;
import org.apache.pinot.spi.data.TimeGranularitySpec.TimeFormat;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.JsonUtils;
@@ -43,12 +33,12 @@ import org.testng.annotations.Test;
import org.testng.collections.Lists;
+@SuppressWarnings("deprecation")
public class SchemaTest {
public static final Logger LOGGER =
LoggerFactory.getLogger(SchemaTest.class);
@Test
- public void testValidation()
- throws Exception {
+ public void testValidation() {
Schema schemaToValidate;
schemaToValidate = new Schema();
@@ -296,32 +286,6 @@ public class SchemaTest {
Assert.assertEquals(schema11, schema12);
}
- @Test
- public void testSerializeDeserialize()
- throws Exception {
- URL resourceUrl =
getClass().getClassLoader().getResource("schemaTest.schema");
- Assert.assertNotNull(resourceUrl);
- Schema schema = Schema.fromFile(new File(resourceUrl.getFile()));
-
- Schema schemaToCompare = Schema.fromString(schema.toPrettyJsonString());
- Assert.assertEquals(schemaToCompare, schema);
- Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode());
-
- schemaToCompare = Schema.fromString(schema.toSingleLineJsonString());
- Assert.assertEquals(schemaToCompare, schema);
- Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode());
-
- schemaToCompare = SchemaUtils.fromZNRecord(SchemaUtils.toZNRecord(schema));
- Assert.assertEquals(schemaToCompare, schema);
- Assert.assertEquals(schemaToCompare.hashCode(), schema.hashCode());
-
- // When setting new fields, schema string should be updated
- String jsonSchema = schemaToCompare.toSingleLineJsonString();
- schemaToCompare.setSchemaName("newSchema");
- String jsonSchemaToCompare = schemaToCompare.toSingleLineJsonString();
- Assert.assertNotEquals(jsonSchemaToCompare, jsonSchema);
- }
-
@Test
public void testSerializeDeserializeOptions()
throws IOException {
@@ -358,13 +322,9 @@ public class SchemaTest {
}
@Test
- public void testTimestampFormatOverride()
- throws Exception {
- URL resourceUrl =
getClass().getClassLoader().getResource("schemaTest.schema");
- Assert.assertNotNull(resourceUrl);
- Schema schema = Schema.fromFile(new File(resourceUrl.getFile()));
- DateTimeFieldSpec fieldSpec = schema.getDateTimeSpec("dateTime3");
- Assert.assertNotNull(fieldSpec);
+ public void testTimestampFormatOverride() {
+ DateTimeFieldSpec fieldSpec =
+ new DateTimeFieldSpec("dateTime", FieldSpec.DataType.TIMESTAMP,
"1:MILLISECONDS:EPOCH", "1:SECONDS");
Assert.assertEquals(fieldSpec.getFormat(), "TIMESTAMP");
}
@@ -392,6 +352,178 @@ public class SchemaTest {
Assert.assertEquals(actualSchema.hashCode(), expectedSchema.hashCode());
}
+ @Test
+ public void testConversionFromTimeToDateTimeSpec() {
+ TimeFieldSpec timeFieldSpec;
+ DateTimeFieldSpec expectedDateTimeFieldSpec;
+ DateTimeFieldSpec actualDateTimeFieldSpec;
+
+ /* 1] only incoming */
+
+ // incoming epoch millis
+ timeFieldSpec =
+ new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS, "incoming"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("incoming", FieldSpec.DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // incoming epoch hours
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.HOURS, "incoming"));
+ expectedDateTimeFieldSpec = new DateTimeFieldSpec("incoming",
FieldSpec.DataType.INT, "1:HOURS:EPOCH", "1:HOURS");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // Simple date format
+ timeFieldSpec = new TimeFieldSpec(
+ new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyyMMdd", "incoming"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("incoming", FieldSpec.DataType.INT,
"1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", "1:DAYS");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // simple date format STRING
+ timeFieldSpec = new TimeFieldSpec(
+ new TimeGranularitySpec(FieldSpec.DataType.STRING, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyy-MM-dd hh-mm-ss",
+ "incoming"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("incoming", FieldSpec.DataType.STRING,
"1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd hh-mm-ss",
+ "1:DAYS");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // time unit size
+ timeFieldSpec =
+ new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG, 5,
TimeUnit.MINUTES, "incoming"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("incoming", FieldSpec.DataType.LONG,
"5:MINUTES:EPOCH", "5:MINUTES");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // transform function
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.HOURS, "incoming"));
+ timeFieldSpec.setTransformFunction("toEpochHours(timestamp)");
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("incoming", FieldSpec.DataType.INT,
"1:HOURS:EPOCH", "1:HOURS", null,
+ "toEpochHours(timestamp)");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ /* 2] incoming + outgoing */
+
+ // same incoming and outgoing
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS, "time"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS,
"time"));
+ expectedDateTimeFieldSpec = new DateTimeFieldSpec("time",
FieldSpec.DataType.LONG, "1:HOURS:EPOCH", "1:HOURS");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // same incoming and outgoing - simple date format
+ timeFieldSpec = new TimeFieldSpec(
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyyMMdd", "time"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyyMMdd", "time"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("time", FieldSpec.DataType.LONG,
"1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd", "1:DAYS");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // millis to hours
+ timeFieldSpec =
+ new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS,
"outgoing"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("outgoing", FieldSpec.DataType.LONG,
"1:HOURS:EPOCH", "1:HOURS", null,
+ "toEpochHours(incoming)");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // millis to bucketed minutes
+ timeFieldSpec =
+ new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, 10,
TimeUnit.MINUTES, "outgoing"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("outgoing", FieldSpec.DataType.LONG,
"10:MINUTES:EPOCH", "10:MINUTES", null,
+ "toEpochMinutesBucket(incoming, 10)");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // days to millis
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS, "outgoing"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("outgoing", FieldSpec.DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS", null,
+ "fromEpochDays(incoming)");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // bucketed minutes to millis
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.LONG, 5, TimeUnit.MINUTES, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS, "outgoing"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("outgoing", FieldSpec.DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS", null,
+ "fromEpochMinutesBucket(incoming, 5)");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // hours to days
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.HOURS, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS,
"outgoing"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("outgoing", FieldSpec.DataType.INT,
"1:DAYS:EPOCH", "1:DAYS", null,
+ "toEpochDays(fromEpochHours(incoming))");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // minutes to hours
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MINUTES, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS,
"outgoing"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("outgoing", FieldSpec.DataType.LONG,
"1:HOURS:EPOCH", "1:HOURS", null,
+ "toEpochHours(fromEpochMinutes(incoming))");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // bucketed minutes to days
+ timeFieldSpec =
+ new TimeFieldSpec(new TimeGranularitySpec(FieldSpec.DataType.LONG, 10,
TimeUnit.MINUTES, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.DAYS,
"outgoing"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("outgoing", FieldSpec.DataType.LONG,
"1:DAYS:EPOCH", "1:DAYS", null,
+ "toEpochDays(fromEpochMinutesBucket(incoming, 10))");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // seconds to bucketed minutes
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, 5, TimeUnit.MINUTES,
"outgoing"));
+ expectedDateTimeFieldSpec =
+ new DateTimeFieldSpec("outgoing", FieldSpec.DataType.LONG,
"5:MINUTES:EPOCH", "5:MINUTES", null,
+ "toEpochMinutesBucket(fromEpochSeconds(incoming), 5)");
+ actualDateTimeFieldSpec = Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.assertEquals(actualDateTimeFieldSpec, expectedDateTimeFieldSpec);
+
+ // simple date format to millis
+ timeFieldSpec = new TimeFieldSpec(
+ new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.DAYS,
"SIMPLE_DATE_FORMAT:yyyyMMdd", "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.LONG,
TimeUnit.MILLISECONDS, "outgoing"));
+ try {
+ Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.fail();
+ } catch (Exception e) {
+ // expected
+ }
+
+ // hours to simple date format
+ timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.HOURS, "incoming"),
+ new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.HOURS,
"SIMPLE_DATE_FORMAT:yyyyMMddhh", "outgoing"));
+ try {
+ Schema.convertToDateTimeFieldSpec(timeFieldSpec);
+ Assert.fail();
+ } catch (Exception e) {
+ // expected
+ }
+ }
+
@Test
public void testSchemaBackwardCompatibility() {
Schema oldSchema = new
Schema.SchemaBuilder().addSingleValueDimension("svDimension",
FieldSpec.DataType.INT)
@@ -402,6 +534,7 @@ public class SchemaTest {
.addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG,
TimeUnit.DAYS, "time"), null)
.addDateTime("dateTime", FieldSpec.DataType.LONG, "1:HOURS:EPOCH",
"1:HOURS").build();
+ //noinspection DataFlowIssue
Assert.assertThrows(NullPointerException.class, () ->
oldSchema.isBackwardCompatibleWith(null));
// remove column
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
