This is an automated email from the ASF dual-hosted git repository.
xhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new cd75e12 [TE] Remove the offline suffix for offline pinot table.
(#4492)
cd75e12 is described below
commit cd75e12286ab8539e68ea4075b5e69f4c7fc2c85
Author: Xiaohui Sun <[email protected]>
AuthorDate: Mon Aug 5 13:46:30 2019 -0700
[TE] Remove the offline suffix for offline pinot table. (#4492)
* [TE] Remove the offline suffix for offline pinot table.
Currently Pinot doesn't need this to differentiate online/offline tables.
* Disable state transition stats verification in rebalancer cluster
integration test (#4490)
Disable state transition stats verification in
rebalancer cluster integration test as they are
potentially flakey and reliant on the timing.
These stats are reliant on the fact that state
transition has happened (callback invoked on the segment
server) for the server that lost a segment as part
of rebalancing.
While the rebalancer waits for all the ideal state
changes to get reflected in external view, we don't yet
check if the servers have indeed dropped the segments. So
it is possible that server may not have done so by the time
test finishes and thus we sometimes notice the segment dropped
stats off by 1.
Disabling these to keep tests happy and until we make the external
view check stronger by ensuring that it doesn't have any extra
servers for a segment indicating that servers that were supposed
to drop segments as part of rebalancing have truly done so.
* [Instance Assignment] Add configs for instance assignment (#4489)
- Also enhance JsonUtils to support TypeReference for deserialization
* Adding kafka 2.0 doc for using simple consumer (#4493)
* [TE] fix unit tests
---
.../pinot/thirdeye/datalayer/pojo/DatasetConfigBean.java | 1 -
.../datasource/pinot/PinotDataSourceMaxTime.java | 7 +++----
.../datasource/pinot/PinotThirdEyeDataSource.java | 6 ++----
.../apache/pinot/thirdeye/datasource/pinot/PqlUtils.java | 9 ++-------
.../org/apache/pinot/thirdeye/util/ThirdEyeUtils.java | 16 ----------------
.../pinot/thirdeye/datasource/pinot/PqlUtilsTest.java | 4 ++--
6 files changed, 9 insertions(+), 34 deletions(-)
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DatasetConfigBean.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DatasetConfigBean.java
index b8aeba2..1599ab0 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DatasetConfigBean.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DatasetConfigBean.java
@@ -37,7 +37,6 @@ public class DatasetConfigBean extends AbstractBean {
public static final String DEFAULT_COMPLETENESS_ALGORITHM =
Wo4WAvgDataCompletenessAlgorithm.class.getName();
public static String DEFAULT_PREAGGREGATED_DIMENSION_VALUE = "all";
- public static String DATASET_OFFLINE_PREFIX = "_OFFLINE";
public static TimeGranularity DEFAULT_HOURLY_EXPECTED_DELAY = new
TimeGranularity(8, TimeUnit.HOURS);
public static TimeGranularity DEFAULT_DAILY_EXPECTED_DELAY = new
TimeGranularity(36, TimeUnit.HOURS);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotDataSourceMaxTime.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotDataSourceMaxTime.java
index 23abbca..b2b3649 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotDataSourceMaxTime.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotDataSourceMaxTime.java
@@ -66,14 +66,13 @@ public class PinotDataSourceMaxTime {
try {
DatasetConfigDTO datasetConfig =
DAO_REGISTRY.getDatasetConfigDAO().findByDataset(dataset);
// By default, query only offline, unless dataset has been marked as
realtime
- String tableName = ThirdEyeUtils.computeTableName(dataset);
TimeSpec timeSpec =
ThirdEyeUtils.getTimestampTimeSpecFromDatasetConfig(datasetConfig);
long cutoffTime = System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1);
String timeClause = PqlUtils.getBetweenClause(new DateTime(0,
DateTimeZone.UTC), new DateTime(cutoffTime, DateTimeZone.UTC), timeSpec,
dataset);
- String maxTimePql = String.format(COLLECTION_MAX_TIME_QUERY_TEMPLATE,
timeSpec.getColumnName(), tableName, timeClause);
- PinotQuery maxTimePinotQuery = new PinotQuery(maxTimePql, tableName);
+ String maxTimePql = String.format(COLLECTION_MAX_TIME_QUERY_TEMPLATE,
timeSpec.getColumnName(), dataset, timeClause);
+ PinotQuery maxTimePinotQuery = new PinotQuery(maxTimePql, dataset);
ThirdEyeResultSetGroup resultSetGroup;
final long tStart = System.nanoTime();
@@ -87,7 +86,7 @@ public class PinotDataSourceMaxTime {
}
if (resultSetGroup.size() == 0 || resultSetGroup.get(0).getRowCount() ==
0) {
- LOGGER.error("Failed to get latest max time for dataset {} with PQL:
{}", tableName, maxTimePinotQuery.getQuery());
+ LOGGER.error("Failed to get latest max time for dataset {} with PQL:
{}", dataset, maxTimePinotQuery.getQuery());
this.collectionToPrevMaxDataTimeMap.remove(dataset);
} else {
DateTimeZone timeZone = Utils.getDataTimeZone(dataset);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
index 917e0fc..95385a7 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
@@ -169,9 +169,7 @@ public class PinotThirdEyeDataSource implements
ThirdEyeDataSource {
datasetConfig.getPreAggregatedKeyword());
}
- // By default, query only offline, unless dataset has been marked as
realtime
- String tableName = ThirdEyeUtils.computeTableName(dataset);
- String pql = null;
+ String pql;
MetricConfigDTO metricConfig = metricFunction.getMetricConfig();
if (metricConfig != null && metricConfig.isDimensionAsMetric()) {
pql = PqlUtils.getDimensionAsMetricPql(request, metricFunction,
decoratedFilterSet, dataTimeSpec,
@@ -183,7 +181,7 @@ public class PinotThirdEyeDataSource implements
ThirdEyeDataSource {
ThirdEyeResultSetGroup resultSetGroup;
final long tStartFunction = System.nanoTime();
try {
- resultSetGroup = this.executePQL(new PinotQuery(pql, tableName));
+ resultSetGroup = this.executePQL(new PinotQuery(pql, dataset));
if (metricConfig != null) {
ThirdeyeMetricsUtil.getRequestLog()
.success(this.getName(), metricConfig.getDataset(),
metricConfig.getName(), tStartFunction, System.nanoTime());
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PqlUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PqlUtils.java
index 03c86d9..03f1e15 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PqlUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PqlUtils.java
@@ -104,9 +104,7 @@ public class PqlUtils {
StringBuilder sb = new StringBuilder();
String selectionClause = getSelectionClause(metricConfig, metricFunction);
- String tableName = ThirdEyeUtils.computeTableName(dataset);
-
- sb.append("SELECT ").append(selectionClause).append(" FROM
").append(tableName);
+ sb.append("SELECT ").append(selectionClause).append(" FROM
").append(dataset);
String betweenClause = getBetweenClause(startTime, endTimeExclusive,
dataTimeSpec, dataset);
sb.append(" WHERE ").append(betweenClause);
@@ -197,10 +195,7 @@ public class PqlUtils {
StringBuilder sb = new StringBuilder();
String selectionClause =
getDimensionAsMetricSelectionClause(metricFunction, metricValuesColumn);
-
- String tableName = ThirdEyeUtils.computeTableName(dataset);
-
- sb.append("SELECT ").append(selectionClause).append(" FROM
").append(tableName);
+ sb.append("SELECT ").append(selectionClause).append(" FROM
").append(dataset);
String betweenClause = getBetweenClause(startTime, endTimeExclusive,
dataTimeSpec, dataset);
sb.append(" WHERE ").append(betweenClause);
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/ThirdEyeUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/ThirdEyeUtils.java
index b334c2a..52ff174 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/ThirdEyeUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/util/ThirdEyeUtils.java
@@ -338,22 +338,6 @@ public abstract class ThirdEyeUtils {
return alias;
}
-
- //By default, query only offline, unless dataset has been marked as realtime
- public static String computeTableName(String collection) {
- String dataset = null;
- try {
- DatasetConfigDTO datasetConfig =
CACHE_REGISTRY.getDatasetConfigCache().get(collection);
- dataset = collection + DatasetConfigBean.DATASET_OFFLINE_PREFIX;
- if (datasetConfig.isRealtime()) {
- dataset = collection;
- }
- } catch (ExecutionException e) {
- LOG.error("Exception in getting dataset name {}", collection, e);
- }
- return dataset;
- }
-
public static Period getbaselineOffsetPeriodByMode(COMPARE_MODE compareMode)
{
int numWeeksAgo = 1;
switch (compareMode) {
diff --git
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datasource/pinot/PqlUtilsTest.java
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datasource/pinot/PqlUtilsTest.java
index b5cef7c..df68ca5 100644
---
a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datasource/pinot/PqlUtilsTest.java
+++
b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datasource/pinot/PqlUtilsTest.java
@@ -185,7 +185,7 @@ public class PqlUtilsTest {
String pql = PqlUtils.getPql(request, metricFunction,
ArrayListMultimap.<String, String>create(), timeSpec);
- Assert.assertEquals(pql, "SELECT AVG(metric) FROM collection_OFFLINE WHERE
metric >= 1 AND metric < 2 GROUP BY dimension TOP 12345");
+ Assert.assertEquals(pql, "SELECT AVG(metric) FROM collection WHERE metric
>= 1 AND metric < 2 GROUP BY dimension TOP 12345");
}
@Test
@@ -203,6 +203,6 @@ public class PqlUtilsTest {
String pql = PqlUtils.getPql(request, metricFunction,
ArrayListMultimap.<String, String>create(), timeSpec);
- Assert.assertEquals(pql, "SELECT AVG(metric) FROM collection_OFFLINE WHERE
metric >= 1 AND metric < 2 GROUP BY dimension TOP 100000");
+ Assert.assertEquals(pql, "SELECT AVG(metric) FROM collection WHERE metric
>= 1 AND metric < 2 GROUP BY dimension TOP 100000");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
