This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 56ef680 Fix an issue with datetime column in Rule Engine (#6731)
56ef680 is described below
commit 56ef6803a962ad5f51295d9ff8194cb29f8fc47c
Author: Sajjad Moradi <[email protected]>
AuthorDate: Wed Mar 31 22:31:21 2021 -0700
Fix an issue with datetime column in Rule Engine (#6731)
* Fix an issue with datetime column in Rule Engine
* minor refactoring
* public -> private
---
.../pinot/controller/recommender/io/InputManager.java | 18 +++++++++++++-----
.../pinot/controller/recommender/TestConfigEngine.java | 13 +++++++++++--
...n => RealtimeProvisioningInput_dateTimeColumn.json} | 14 +++++++-------
....json => RealtimeProvisioningInput_timeColumn.json} | 0
4 files changed, 31 insertions(+), 14 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/io/InputManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/io/InputManager.java
index 69f2694..d6e3ccb 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/io/InputManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/recommender/io/InputManager.java
@@ -41,6 +41,7 @@ import
org.apache.pinot.core.query.request.context.utils.BrokerRequestToQueryCon
import org.apache.pinot.core.requesthandler.BrokerRequestOptimizer;
import org.apache.pinot.core.requesthandler.PinotQueryParserFactory;
import org.apache.pinot.parsers.AbstractCompiler;
+import org.apache.pinot.spi.data.DateTimeFieldSpec;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.MetricFieldSpec;
@@ -141,7 +142,7 @@ public class InputManager {
throws InvalidInputException {
LOGGER.info("Preprocessing Input:");
reorderDimsAndBuildMap();
- registerColnameFieldType();
+ registerColNameFieldType();
validateQueries();
if (_useCardinalityNormalization){
regulateCardinalityForAll();
@@ -181,15 +182,20 @@ public class InputManager {
invalidQueries.forEach(_queryWeightMap::remove);
}
- public void registerColnameFieldType() { // create a map from colname to
data type
+ // create a map from col name to data type
+ private void registerColNameFieldType() {
for (DimensionFieldSpec dimensionFieldSpec :
_schema.getDimensionFieldSpecs()) {
_colNameFieldTypeMap.put(dimensionFieldSpec.getName(),
dimensionFieldSpec.getDataType());
}
for (MetricFieldSpec metricFieldSpec : _schema.getMetricFieldSpecs()) {
_colNameFieldTypeMap.put(metricFieldSpec.getName(),
metricFieldSpec.getDataType());
}
- //TODO: add support for multiple getDateTimeFieldSpecs
- _colNameFieldTypeMap.put(_schema.getTimeFieldSpec().getName(),
_schema.getTimeFieldSpec().getDataType());
+ for (DateTimeFieldSpec dateTimeFieldSpec :
_schema.getDateTimeFieldSpecs()) {
+ _colNameFieldTypeMap.put(dateTimeFieldSpec.getName(),
dateTimeFieldSpec.getDataType());
+ }
+ if (_schemaWithMetaData.getTimeFieldSpec() != null) {
+ _colNameFieldTypeMap.put(_schema.getTimeFieldSpec().getName(),
_schema.getTimeFieldSpec().getDataType());
+ }
}
private void reorderDimsAndBuildMap()
@@ -361,7 +367,9 @@ public class InputManager {
_schemaWithMetaData.getDateTimeFieldSpecs().forEach(fieldMetadata -> {
_metaDataMap.put(fieldMetadata.getName(), fieldMetadata);
});
- _metaDataMap.put(_schemaWithMetaData.getTimeFieldSpec().getName(),
_schemaWithMetaData.getTimeFieldSpec());
+ if (_schemaWithMetaData.getTimeFieldSpec() != null) {
+ _metaDataMap.put(_schemaWithMetaData.getTimeFieldSpec().getName(),
_schemaWithMetaData.getTimeFieldSpec());
+ }
}
@JsonIgnore
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/TestConfigEngine.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/TestConfigEngine.java
index c241543..257142c 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/TestConfigEngine.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/recommender/TestConfigEngine.java
@@ -361,8 +361,17 @@ public class TestConfigEngine {
}
@Test
- void testRealtimeProvisioningRule() throws Exception {
- String input =
readInputToStr("recommenderInput/RealtimeProvisioningInput.json");
+ void testRealtimeProvisioningRule_withTimeColumn() throws Exception {
+
testRealtimeProvisioningRule("recommenderInput/RealtimeProvisioningInput_timeColumn.json");
+ }
+
+ @Test
+ void testRealtimeProvisioningRule_withDateTimeColumn() throws Exception {
+
testRealtimeProvisioningRule("recommenderInput/RealtimeProvisioningInput_dateTimeColumn.json");
+ }
+
+ private void testRealtimeProvisioningRule(String fileName) throws Exception {
+ String input = readInputToStr(fileName);
String output = RecommenderDriver.run(input);
ConfigManager configManager = objectMapper.readValue(output,
ConfigManager.class);
Map<String, Map<String, String>> recommendations =
configManager.getRealtimeProvisioningRecommendations();
diff --git
a/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput.json
b/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_dateTimeColumn.json
similarity index 96%
copy from
pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput.json
copy to
pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_dateTimeColumn.json
index cd568d5..a846ecb 100644
---
a/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput.json
+++
b/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_dateTimeColumn.json
@@ -116,15 +116,15 @@
"numValuesPerEntry":1
}
],
- "timeFieldSpec": {
- "incomingGranularitySpec": {
- "dataType": "INT",
+ "dateTimeFieldSpecs": [
+ {
"name": "t",
- "timeType": "DAYS",
- "cardinality":1000,
- "numValuesPerEntry":1
+ "dataType": "INT",
+ "format": "1:DAYS:EPOCH",
+ "granularity": "1:DAYS",
+ "cardinality": 1000
}
- }
+ ]
},
"queriesWithWeights":{
"select i from tableName where b in (2,4) and ((a in (1,2,3) and e = 4) or
c = 7) and d in ('#VALUES', 23) and t > 500": 1,
diff --git
a/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput.json
b/pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_timeColumn.json
similarity index 100%
rename from
pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput.json
rename to
pinot-controller/src/test/resources/recommenderInput/RealtimeProvisioningInput_timeColumn.json
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
