This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 94aad60 Include transformFunction in timeFieldSpec toJsonObject
(#5464)
94aad60 is described below
commit 94aad60f520d627e257cac6c57c54abeafec37a2
Author: Neha Pawar <[email protected]>
AuthorDate: Thu May 28 14:11:08 2020 -0700
Include transformFunction in timeFieldSpec toJsonObject (#5464)
---
.../apache/pinot/common/data/FieldSpecTest.java | 10 ++++++---
pinot-common/src/test/resources/schemaTest.schema | 25 +++++++++++++++++++---
.../java/org/apache/pinot/spi/data/FieldSpec.java | 10 ++++++---
.../org/apache/pinot/spi/data/TimeFieldSpec.java | 1 +
4 files changed, 37 insertions(+), 9 deletions(-)
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
index 9d29bd4..8063988 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/data/FieldSpecTest.java
@@ -262,7 +262,7 @@ public class FieldSpecTest {
FieldSpec second;
// Single-value boolean type dimension field with default null value.
- String[] dimensionFields = {"\"name\":\"dimension\"",
"\"dataType\":\"BOOLEAN\"", "\"defaultNullValue\":false"};
+ String[] dimensionFields = {"\"name\":\"dimension\"",
"\"dataType\":\"BOOLEAN\"", "\"defaultNullValue\":false",
"\"transformFunction\":\"trim(foo)\""};
first =
JsonUtils.stringToObject(getRandomOrderJsonString(dimensionFields),
DimensionFieldSpec.class);
second = JsonUtils.stringToObject(first.toJsonObject().toString(),
DimensionFieldSpec.class);
Assert.assertEquals(first, second, ERROR_MESSAGE);
@@ -276,14 +276,18 @@ public class FieldSpecTest {
// Time field with default value.
String[] timeFields =
- {"\"incomingGranularitySpec\":{\"timeUnitSize\":1,
\"timeType\":\"MILLISECONDS\",\"dataType\":\"LONG\",\"name\":\"incomingTime\"}",
"\"outgoingGranularitySpec\":{\"timeType\":\"SECONDS\",\"dataType\":\"INT\",\"name\":\"outgoingTime\"}",
"\"defaultNullValue\":-1"};
+ {"\"incomingGranularitySpec\":{\"timeUnitSize\":1,
\"timeType\":\"MILLISECONDS\",\"dataType\":\"LONG\",\"name\":\"incomingTime\"}",
+
"\"outgoingGranularitySpec\":{\"timeType\":\"SECONDS\",\"dataType\":\"INT\",\"name\":\"outgoingTime\"}",
+ "\"defaultNullValue\":-1",
+ "\"transformFunction\":\"toEpochDays(millis)\""};
first = JsonUtils.stringToObject(getRandomOrderJsonString(timeFields),
TimeFieldSpec.class);
second = JsonUtils.stringToObject(first.toJsonObject().toString(),
TimeFieldSpec.class);
Assert.assertEquals(first, second, ERROR_MESSAGE);
// DateTime field
String[] dateTimeFields =
- {"\"name\":\"Date\"", "\"dataType\":\"LONG\"",
"\"format\":\"1:MILLISECONDS:EPOCH\"", "\"granularity\":\"5:MINUTES\"",
"\"dateTimeType\":\"PRIMARY\""};
+ {"\"name\":\"Date\"", "\"dataType\":\"LONG\"",
"\"format\":\"1:MILLISECONDS:EPOCH\"", "\"granularity\":\"5:MINUTES\"",
+ "\"transformFunction\":\"fromEpochDays(daysSinceEpoch)\""};
first = JsonUtils.stringToObject(getRandomOrderJsonString(dateTimeFields),
DateTimeFieldSpec.class);
second = JsonUtils.stringToObject(first.toJsonObject().toString(),
DateTimeFieldSpec.class);
Assert.assertEquals(first, second, ERROR_MESSAGE);
diff --git a/pinot-common/src/test/resources/schemaTest.schema
b/pinot-common/src/test/resources/schemaTest.schema
index 32a83c0..b7c7dcf 100644
--- a/pinot-common/src/test/resources/schemaTest.schema
+++ b/pinot-common/src/test/resources/schemaTest.schema
@@ -8,7 +8,8 @@
{
"name": "longMetric",
"dataType": "LONG",
- "singleValueField": true
+ "singleValueField": true,
+ "transformFunction": "Groovy({foo/1000}, foo)"
},
{
"name": "floatMetric",
@@ -37,7 +38,8 @@
"name": "longDimension",
"dataType": "LONG",
"singleValueField": false,
- "delimiter": null
+ "delimiter": null,
+ "transformFunction": "trim(bar)"
},
{
"name": "floatDimension",
@@ -69,7 +71,24 @@
"timeType": "DAYS",
"name": "outgoing"
},
- "defaultNullValue": 10000
+ "defaultNullValue": 10000,
+ "transformFunction": "toEpochDays(millis)"
},
+ "dateTimeFieldSpecs": [
+ {
+ "name": "dateTime1",
+ "dataType": "LONG",
+ "format": "1:MILLISECONDS:EPOCH",
+ "granularity": "15:MINUTES",
+ "transformFunction": "fromDateTime(dt, 'yyyyMMdd')"
+ },
+ {
+ "name": "dateTime2",
+ "dataType": "STRING",
+ "format": "1:DAYS:SIMPLE_DATE_FORMAT:yyyyMMdd",
+ "granularity": "1:DAYS",
+ "defaultNullValue": "20200101"
+ }
+ ],
"schemaName": "schemaTest"
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
index de3e3ee..aabbd7f 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
@@ -263,9 +263,7 @@ public abstract class FieldSpec implements
Comparable<FieldSpec> {
jsonObject.put("maxLength", _maxLength);
}
appendDefaultNullValue(jsonObject);
- if (_transformFunction != null) {
- jsonObject.put("transformFunction", _transformFunction);
- }
+ appendTransformFunction(jsonObject);
return jsonObject;
}
@@ -280,6 +278,12 @@ public abstract class FieldSpec implements
Comparable<FieldSpec> {
}
}
+ protected void appendTransformFunction(ObjectNode jsonNode) {
+ if (_transformFunction != null) {
+ jsonNode.put("transformFunction", _transformFunction);
+ }
+ }
+
@SuppressWarnings("EqualsWhichDoesntCheckParameterClass")
@Override
public boolean equals(Object o) {
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
index 5fefbf7..1699ab8 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
@@ -112,6 +112,7 @@ public final class TimeFieldSpec extends FieldSpec {
jsonObject.set("outgoingGranularitySpec",
_outgoingGranularitySpec.toJsonObject());
}
appendDefaultNullValue(jsonObject);
+ appendTransformFunction(jsonObject);
return jsonObject;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
