This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4b4224f [TE] rootcause - baseline monthly, daily, hourly support
(#3585)
4b4224f is described below
commit 4b4224f3221d8c2ca40ef1a01541095af7b7a259
Author: Alexander Pucher <[email protected]>
AuthorDate: Wed Dec 5 12:25:34 2018 -0800
[TE] rootcause - baseline monthly, daily, hourly support (#3585)
This PR adds support for other month-over-month, day-over-day, and
hour-over-hour baselines in the RCA GUI. The UI changes are unobtrusive to
enable us to pilot these changes with select users
---
.../rootcause-select-comparison-range/component.js | 39 ++++---
.../thirdeye-frontend/app/pods/rootcause/route.js | 2 +-
thirdeye/thirdeye-frontend/app/utils/rca-utils.js | 17 ++-
.../resources/v2/BaselineParsingUtils.java | 115 ++++++++++++++++++---
.../thirdeye/dataframe/util/DataFrameUtils.java | 12 +++
.../rootcause/timeseries/BaselineAggregate.java | 74 +++++++++++++
6 files changed, 228 insertions(+), 31 deletions(-)
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/rootcause-select-comparison-range/component.js
b/thirdeye/thirdeye-frontend/app/pods/components/rootcause-select-comparison-range/component.js
index 8eddc58..0367de6 100644
---
a/thirdeye/thirdeye-frontend/app/pods/components/rootcause-select-comparison-range/component.js
+++
b/thirdeye/thirdeye-frontend/app/pods/components/rootcause-select-comparison-range/component.js
@@ -2,8 +2,7 @@ import { computed } from '@ember/object';
import Component from '@ember/component';
import $ from 'jquery';
import {
- makeTime,
- dateFormatFull
+ makeTime
} from 'thirdeye-frontend/utils/rca-utils';
// TODO consolidate rootcause-select-comparison-range2, rootcause-slider
@@ -31,7 +30,7 @@ const namedToEpocMapping = {
export default Component.extend({
- timeFormat: "MMM D, hh:mm a z",//slider
+ timeFormat: "MMM D, hh:mm a z", //slider
range: null, // [0, 0]
compareMode: null, // ""
onChange: null, // func (start, end, compareMode)
@@ -47,16 +46,26 @@ export default Component.extend({
},
compareModeOptions: [
- 'WoW',
- 'Wo2W',
- 'Wo3W',
- 'Wo4W',
- 'mean4w',
- 'median4w',
- 'min4w',
- 'max4w',
- 'predicted',
- 'none'
+ {
+ groupName: 'Weekly',
+ options: [ 'wo1w', 'wo2w', 'wo3w', 'wo4w', 'mean4w', 'median4w',
'min4w', 'max4w' ]
+ },
+ {
+ groupName: 'Hourly',
+ options: [ 'ho1h', 'ho2h', 'ho3h', 'ho6h', 'mean6h', 'median6h',
'min6h', 'max6h' ]
+ },
+ {
+ groupName: 'Daily',
+ options: [ 'do1d', 'do2d', 'do3d', 'do4d', 'mean4d', 'median4d',
'min4d', 'max4d' ]
+ },
+ {
+ groupName: 'Monthly',
+ options: [ 'mo1m', 'mo2m', 'mo3m', 'mo6m', 'mean6m', 'median6m',
'min6m', 'max6m' ]
+ },
+ {
+ groupName: 'Algorithm',
+ options: [ 'predicted', 'none' ]
+ }
],
minDisplayWindow: computed('displayRange.[]', function() {//display window
start - slider
@@ -135,7 +144,7 @@ export default Component.extend({
this.setProperties({
startFormattedOneWay: this.get('startFormatted'),
endFormattedOneWay: this.get('endFormatted'),
- maxDateFormattedOneWay: this.get('maxDateFormatted'),
+ maxDateFormattedOneWay: this.get('maxDateFormatted')
// granularityOneWay: this.get('granularity')
});
@@ -166,7 +175,7 @@ export default Component.extend({
to: this.get('maxInvestigatePeriod').format('x')
});
- // Update the display window's investiation period on the chart
+ // Update the display window's investigation period on the chart
onChange(makeTime(start).valueOf(), makeTime(end).valueOf(),
compareMode);
},
diff --git a/thirdeye/thirdeye-frontend/app/pods/rootcause/route.js
b/thirdeye/thirdeye-frontend/app/pods/rootcause/route.js
index e4ca2d3..1cd28ed 100644
--- a/thirdeye/thirdeye-frontend/app/pods/rootcause/route.js
+++ b/thirdeye/thirdeye-frontend/app/pods/rootcause/route.js
@@ -255,7 +255,7 @@ export default Route.extend(AuthenticatedRouteMixin, {
makeTime().startOf('day').add(1, 'day').valueOf()
],
granularity: '1_HOURS',
- compareMode: 'WoW'
+ compareMode: 'wo1w'
};
// default params
diff --git a/thirdeye/thirdeye-frontend/app/utils/rca-utils.js
b/thirdeye/thirdeye-frontend/app/utils/rca-utils.js
index ca16f86..b8b47ea 100644
--- a/thirdeye/thirdeye-frontend/app/utils/rca-utils.js
+++ b/thirdeye/thirdeye-frontend/app/utils/rca-utils.js
@@ -395,7 +395,22 @@ export function toBaselineRange(range, offset) {
mean4w: 1, // default. not fully supported by backend yet
median4w: 1, // default. not fully supported by backend yet
min4w: 1, // default. not fully supported by backend yet
- max4w: 1 // default. not fully supported by backend yet
+ max4w: 1, // default. not fully supported by backend yet
+ ho1h: 1,
+ ho2h: 1,
+ ho3h: 1,
+ median4h: 1,
+ mean4h: 1,
+ do1d: 1,
+ do2d: 1,
+ do3d: 1,
+ median4d: 1,
+ mean4d: 1,
+ mo1m: 1,
+ mo2m: 1,
+ mo3m: 1,
+ median4m: 1,
+ mean4m: 1
}[offset.toLowerCase()];
if (offsetWeeks === 0) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/v2/BaselineParsingUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/v2/BaselineParsingUtils.java
index 1ff9293..56f643a 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/v2/BaselineParsingUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/v2/BaselineParsingUtils.java
@@ -29,11 +29,36 @@ public class BaselineParsingUtils {
private static final Pattern PATTERN_NONE = Pattern.compile("none");
private static final Pattern PATTERN_PREDICTED =
Pattern.compile("predicted");
private static final Pattern PATTERN_CURRENT = Pattern.compile("current");
+ private static final Pattern PATTERN_HOUR_OVER_HOUR =
Pattern.compile("ho([1-9][0-9]*)h");
+ private static final Pattern PATTERN_DAY_OVER_DAY =
Pattern.compile("do([1-9][0-9]*)d");
private static final Pattern PATTERN_WEEK_OVER_WEEK =
Pattern.compile("wo([1-9][0-9]*)w");
- private static final Pattern PATTERN_MEAN =
Pattern.compile("mean([1-9][0-9]*)w");
- private static final Pattern PATTERN_MEDIAN =
Pattern.compile("median([1-9][0-9]*)w");
- private static final Pattern PATTERN_MIN =
Pattern.compile("min([1-9][0-9]*)w");
- private static final Pattern PATTERN_MAX =
Pattern.compile("max([1-9][0-9]*)w");
+ private static final Pattern PATTERN_MONTH_OVER_MONTH =
Pattern.compile("mo([1-9][0-9]*)m");
+ private static final Pattern PATTERN_MEAN =
Pattern.compile("mean([1-9][0-9]*)(h|d|w|m)");
+ private static final Pattern PATTERN_MEDIAN =
Pattern.compile("median([1-9][0-9]*)(h|d|w|m)");
+ private static final Pattern PATTERN_MIN =
Pattern.compile("min([1-9][0-9]*)(h|d|w|m)");
+ private static final Pattern PATTERN_MAX =
Pattern.compile("max([1-9][0-9]*)(h|d|w|m)");
+
+ private enum Unit {
+ HOUR("h"),
+ DAY("d"),
+ WEEK("w"),
+ MONTH("m");
+
+ final String unit;
+
+ Unit(String unit) {
+ this.unit = unit;
+ }
+
+ static Unit fromUnit(String unit) {
+ for (Unit u : Unit.values()) {
+ if (u.unit.equals(unit)) {
+ return u;
+ }
+ }
+ throw new IllegalArgumentException(String.format("Unknown unit '%s'",
unit));
+ }
+ }
/**
* Returns a configured instance of Baseline for the given, named offset.
The method uses slice and
@@ -43,11 +68,14 @@ public class BaselineParsingUtils {
* <pre>
* current the time range as specified by start and end)
* none empty time range
+ * hoXh hour-over-hour data points with a lag of X hours)
+ * doXd day-over-day data points with a lag of X days)
* woXw week-over-week data points with a lag of X weeks)
- * meanXw average of data points from the the past X weeks, with a lag
of 1 week)
- * medianXw median of data points from the the past X weeks, with a lag
of 1 week)
- * minXw minimum of data points from the the past X weeks, with a lag
of 1 week)
- * maxXw maximum of data points from the the past X weeks, with a lag
of 1 week)
+ * moXm month-over-month data points with a lag of X months)
+ * meanXU average of data points from the the past X units (hour, day,
month, week), with a lag of 1 unit)
+ * medianXU median of data points from the the past X units (hour, day,
month, week), with a lag of 1 unit)
+ * minXU minimum of data points from the the past X units (hour, day,
month, week), with a lag of 1 unit)
+ * maxXU maximum of data points from the the past X units (hour, day,
month, week), with a lag of 1 unit)
* </pre>
*
* @param offset offset identifier
@@ -74,31 +102,90 @@ public class BaselineParsingUtils {
return new BaselineNone();
}
+ BaselineAggregateType agg = null;
+ int shift = Integer.MIN_VALUE;
+ int count = Integer.MIN_VALUE;
+ Unit unit = null;
+
+ Matcher mHourOverHour = PATTERN_HOUR_OVER_HOUR.matcher(offset);
+ if (mHourOverHour.find()) {
+ agg = BaselineAggregateType.SUM;
+ shift = Integer.valueOf(mHourOverHour.group(1));
+ count = 1;
+ unit = Unit.HOUR;
+ }
+
+ Matcher mDayOverDay = PATTERN_DAY_OVER_DAY.matcher(offset);
+ if (mDayOverDay.find()) {
+ agg = BaselineAggregateType.SUM;
+ shift = Integer.valueOf(mDayOverDay.group(1));
+ count = 1;
+ unit = Unit.DAY;
+ }
+
Matcher mWeekOverWeek = PATTERN_WEEK_OVER_WEEK.matcher(offset);
if (mWeekOverWeek.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.SUM, 1,
Integer.valueOf(mWeekOverWeek.group(1)), timeZone);
+ agg = BaselineAggregateType.SUM;
+ shift = Integer.valueOf(mWeekOverWeek.group(1));
+ count = 1;
+ unit = Unit.WEEK;
+ }
+
+ Matcher mMonthOverMonth = PATTERN_MONTH_OVER_MONTH.matcher(offset);
+ if (mMonthOverMonth.find()) {
+ agg = BaselineAggregateType.SUM;
+ shift = Integer.valueOf(mMonthOverMonth.group(1));
+ count = 1;
+ unit = Unit.MONTH;
}
Matcher mMean = PATTERN_MEAN.matcher(offset);
if (mMean.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MEAN,
Integer.valueOf(mMean.group(1)), 1, timeZone);
+ agg = BaselineAggregateType.MEAN;
+ shift = 1;
+ count = Integer.valueOf(mMean.group(1));
+ unit = Unit.fromUnit(mMean.group(2));
}
Matcher mMedian = PATTERN_MEDIAN.matcher(offset);
if (mMedian.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MEDIAN,
Integer.valueOf(mMedian.group(1)), 1, timeZone);
+ agg = BaselineAggregateType.MEDIAN;
+ shift = 1;
+ count = Integer.valueOf(mMedian.group(1));
+ unit = Unit.fromUnit(mMedian.group(2));
}
Matcher mMin = PATTERN_MIN.matcher(offset);
if (mMin.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MIN,
Integer.valueOf(mMin.group(1)), 1, timeZone);
+ agg = BaselineAggregateType.MIN;
+ shift = 1;
+ count = Integer.valueOf(mMin.group(1));
+ unit = Unit.fromUnit(mMin.group(2));
}
Matcher mMax = PATTERN_MAX.matcher(offset);
if (mMax.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MAX,
Integer.valueOf(mMax.group(1)), 1, timeZone);
+ agg = BaselineAggregateType.MAX;
+ shift = 1;
+ count = Integer.valueOf(mMax.group(1));
+ unit = Unit.fromUnit(mMax.group(2));
+ }
+
+ if (agg == null || unit == null || shift == Integer.MIN_VALUE || count ==
Integer.MIN_VALUE ) {
+ throw new IllegalArgumentException(String.format("Unsupported offset
'%s'", offset));
}
- throw new IllegalArgumentException(String.format("Unknown offset '%s'",
offset));
+ switch (unit) {
+ case HOUR:
+ return BaselineAggregate.fromHourOverHour(agg, count, shift, timeZone);
+ case DAY:
+ return BaselineAggregate.fromDayOverDay(agg, count, shift, timeZone);
+ case WEEK:
+ return BaselineAggregate.fromWeekOverWeek(agg, count, shift, timeZone);
+ case MONTH:
+ return BaselineAggregate.fromMonthOverMonth(agg, count, shift,
timeZone);
+ default:
+ throw new IllegalArgumentException(String.format("Unsupported unit
'%s'", unit));
+ }
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/util/DataFrameUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/util/DataFrameUtils.java
index 2d8be7a..36d08f2 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/util/DataFrameUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/util/DataFrameUtils.java
@@ -443,6 +443,13 @@ public class DataFrameUtils {
fields.add(DateTimeFieldType.secondOfMinute());
fields.add(DateTimeFieldType.millisOfSecond());
+ } else if (PeriodType.months().equals(type)) {
+ fields.add(DateTimeFieldType.dayOfMonth());
+ fields.add(DateTimeFieldType.hourOfDay());
+ fields.add(DateTimeFieldType.minuteOfHour());
+ fields.add(DateTimeFieldType.secondOfMinute());
+ fields.add(DateTimeFieldType.millisOfSecond());
+
} else {
throw new IllegalArgumentException(String.format("Unsupported PeriodType
'%s'", type));
}
@@ -450,6 +457,11 @@ public class DataFrameUtils {
int[] zeros = new int[fields.size()];
Arrays.fill(zeros, 0);
+ // workaround for dayOfMonth > 0 constraint
+ if (PeriodType.months().equals(type)) {
+ zeros[0] = 1;
+ }
+
return new Partial(fields.toArray(new DateTimeFieldType[fields.size()]),
zeros);
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/rootcause/timeseries/BaselineAggregate.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/rootcause/timeseries/BaselineAggregate.java
index 4f34804..be929dc 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/rootcause/timeseries/BaselineAggregate.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/rootcause/timeseries/BaselineAggregate.java
@@ -232,6 +232,27 @@ public class BaselineAggregate implements Baseline {
}
/**
+ * Returns an instance of BaselineAggregate for the specified type and
{@code numMonths} offsets
+ * computed on a consecutive month-over-month basis starting with a lag of
{@code offsetMonths}.
+ * <br/><b>NOTE:</b> this will apply DST correction
+ *
+ * @see BaselineAggregateType
+ *
+ * @param type aggregation type
+ * @param numMonths number of consecutive months
+ * @param offsetMonths lag for starting consecutive months
+ * @param timeZone time zone
+ * @return BaselineAggregate with given type and monthly offsets
+ */
+ public static BaselineAggregate fromMonthOverMonth(BaselineAggregateType
type, int numMonths, int offsetMonths, DateTimeZone timeZone) {
+ List<Period> offsets = new ArrayList<>();
+ for (int i = 0; i < numMonths; i++) {
+ offsets.add(new Period(0, -1 * (i + offsetMonths), 0, 0, 0, 0, 0, 0,
PeriodType.months()));
+ }
+ return new BaselineAggregate(type, offsets, timeZone, PeriodType.months());
+ }
+
+ /**
* Returns an instance of BaselineAggregate for the specified type and
{@code numWeeks} offsets
* computed on a consecutive week-over-week basis starting with a lag of
{@code offsetWeeks}.
* <br/><b>NOTE:</b> this will apply DST correction (modeled as 7 days)
@@ -384,6 +405,25 @@ public class BaselineAggregate implements Baseline {
}
};
+ } else if (PeriodType.months().equals(this.periodType)) {
+ return new Series.LongFunction() {
+ @Override
+ public long apply(long... values) {
+ DateTime dateTime = new DateTime(values[0],
BaselineAggregate.this.timeZone);
+ long months = new Period(origin, dateTime,
BaselineAggregate.this.periodType).getMonths();
+ long days = dateTime.getDayOfMonth() - 1; // workaround for
dayOfMonth > 0 constraint
+ if (days == dateTime.dayOfMonth().getMaximumValue() - 1) {
+ days = 99;
+ }
+
+ long hours = dateTime.getHourOfDay();
+ long minutes = dateTime.getMinuteOfHour();
+ long seconds = dateTime.getSecondOfMinute();
+ long millis = dateTime.getMillisOfSecond();
+ return months * 100000000000L + days * 1000000000L + hours *
10000000L + minutes * 100000L + seconds * 1000L + millis;
+ }
+ };
+
} else {
throw new IllegalArgumentException(String.format("Unsupported PeriodType
'%s'", this.periodType));
}
@@ -469,6 +509,40 @@ public class BaselineAggregate implements Baseline {
}
};
+ } else if (PeriodType.months().equals(this.periodType)) {
+ return new Series.LongFunction() {
+ @Override
+ public long apply(long... values) {
+ int months = (int) (values[0] / 100000000000L);
+ int days = (int) ((values[0] / 1000000000L) % 100L);
+ int hours = (int) ((values[0] / 10000000L) % 100L);
+ int minutes = (int) ((values[0] / 100000L) % 100L);
+ int seconds = (int) ((values[0] / 1000L) % 100L);
+ int millis = (int) (values[0] % 1000L);
+
+ DateTime originPlusMonth = origin.plusMonths(months);
+
+ // last day of source month
+ if (days >= 99) {
+ days = originPlusMonth.dayOfMonth().getMaximumValue() - 1;
+ }
+
+ // unsupported destination day (e.g. 31st of Feb)
+ if (originPlusMonth.dayOfMonth().getMaximumValue() <
originPlusMonth.getDayOfMonth() + days) {
+ return LongSeries.NULL;
+ }
+
+ DateTime target = originPlusMonth
+ .plusDays(days)
+ .plusHours(hours)
+ .plusMinutes(minutes)
+ .plusSeconds(seconds)
+ .plusMillis(millis);
+
+ return target.getMillis();
+ }
+ };
+
} else {
throw new IllegalArgumentException(String.format("Unsupported PeriodType
'%s'", this.periodType));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
