apucher closed pull request #3585: [TE] rootcause - baseline monthly, daily,
hourly support
URL: https://github.com/apache/incubator-pinot/pull/3585
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/thirdeye/thirdeye-frontend/app/pods/components/rootcause-select-comparison-range/component.js
b/thirdeye/thirdeye-frontend/app/pods/components/rootcause-select-comparison-range/component.js
index 8eddc58d0c..0367de61d3 100644
---
a/thirdeye/thirdeye-frontend/app/pods/components/rootcause-select-comparison-range/component.js
+++
b/thirdeye/thirdeye-frontend/app/pods/components/rootcause-select-comparison-range/component.js
@@ -2,8 +2,7 @@ import { computed } from '@ember/object';
import Component from '@ember/component';
import $ from 'jquery';
import {
- makeTime,
- dateFormatFull
+ makeTime
} from 'thirdeye-frontend/utils/rca-utils';
// TODO consolidate rootcause-select-comparison-range2, rootcause-slider
@@ -31,7 +30,7 @@ const namedToEpocMapping = {
export default Component.extend({
- timeFormat: "MMM D, hh:mm a z",//slider
+ timeFormat: "MMM D, hh:mm a z", //slider
range: null, // [0, 0]
compareMode: null, // ""
onChange: null, // func (start, end, compareMode)
@@ -47,16 +46,26 @@ export default Component.extend({
},
compareModeOptions: [
- 'WoW',
- 'Wo2W',
- 'Wo3W',
- 'Wo4W',
- 'mean4w',
- 'median4w',
- 'min4w',
- 'max4w',
- 'predicted',
- 'none'
+ {
+ groupName: 'Weekly',
+ options: [ 'wo1w', 'wo2w', 'wo3w', 'wo4w', 'mean4w', 'median4w',
'min4w', 'max4w' ]
+ },
+ {
+ groupName: 'Hourly',
+ options: [ 'ho1h', 'ho2h', 'ho3h', 'ho6h', 'mean6h', 'median6h',
'min6h', 'max6h' ]
+ },
+ {
+ groupName: 'Daily',
+ options: [ 'do1d', 'do2d', 'do3d', 'do4d', 'mean4d', 'median4d',
'min4d', 'max4d' ]
+ },
+ {
+ groupName: 'Monthly',
+ options: [ 'mo1m', 'mo2m', 'mo3m', 'mo6m', 'mean6m', 'median6m',
'min6m', 'max6m' ]
+ },
+ {
+ groupName: 'Algorithm',
+ options: [ 'predicted', 'none' ]
+ }
],
minDisplayWindow: computed('displayRange.[]', function() {//display window
start - slider
@@ -135,7 +144,7 @@ export default Component.extend({
this.setProperties({
startFormattedOneWay: this.get('startFormatted'),
endFormattedOneWay: this.get('endFormatted'),
- maxDateFormattedOneWay: this.get('maxDateFormatted'),
+ maxDateFormattedOneWay: this.get('maxDateFormatted')
// granularityOneWay: this.get('granularity')
});
@@ -166,7 +175,7 @@ export default Component.extend({
to: this.get('maxInvestigatePeriod').format('x')
});
- // Update the display window's investiation period on the chart
+ // Update the display window's investigation period on the chart
onChange(makeTime(start).valueOf(), makeTime(end).valueOf(),
compareMode);
},
diff --git a/thirdeye/thirdeye-frontend/app/pods/rootcause/route.js
b/thirdeye/thirdeye-frontend/app/pods/rootcause/route.js
index e4ca2d329e..1cd28ed2f0 100644
--- a/thirdeye/thirdeye-frontend/app/pods/rootcause/route.js
+++ b/thirdeye/thirdeye-frontend/app/pods/rootcause/route.js
@@ -255,7 +255,7 @@ export default Route.extend(AuthenticatedRouteMixin, {
makeTime().startOf('day').add(1, 'day').valueOf()
],
granularity: '1_HOURS',
- compareMode: 'WoW'
+ compareMode: 'wo1w'
};
// default params
diff --git a/thirdeye/thirdeye-frontend/app/utils/rca-utils.js
b/thirdeye/thirdeye-frontend/app/utils/rca-utils.js
index ca16f86d9f..b8b47ea1fa 100644
--- a/thirdeye/thirdeye-frontend/app/utils/rca-utils.js
+++ b/thirdeye/thirdeye-frontend/app/utils/rca-utils.js
@@ -395,7 +395,22 @@ export function toBaselineRange(range, offset) {
mean4w: 1, // default. not fully supported by backend yet
median4w: 1, // default. not fully supported by backend yet
min4w: 1, // default. not fully supported by backend yet
- max4w: 1 // default. not fully supported by backend yet
+ max4w: 1, // default. not fully supported by backend yet
+ ho1h: 1,
+ ho2h: 1,
+ ho3h: 1,
+ median4h: 1,
+ mean4h: 1,
+ do1d: 1,
+ do2d: 1,
+ do3d: 1,
+ median4d: 1,
+ mean4d: 1,
+ mo1m: 1,
+ mo2m: 1,
+ mo3m: 1,
+ median4m: 1,
+ mean4m: 1
}[offset.toLowerCase()];
if (offsetWeeks === 0) {
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/v2/BaselineParsingUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/v2/BaselineParsingUtils.java
index 1ff92935cf..56f643ac7a 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/v2/BaselineParsingUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/v2/BaselineParsingUtils.java
@@ -29,11 +29,36 @@
private static final Pattern PATTERN_NONE = Pattern.compile("none");
private static final Pattern PATTERN_PREDICTED =
Pattern.compile("predicted");
private static final Pattern PATTERN_CURRENT = Pattern.compile("current");
+ private static final Pattern PATTERN_HOUR_OVER_HOUR =
Pattern.compile("ho([1-9][0-9]*)h");
+ private static final Pattern PATTERN_DAY_OVER_DAY =
Pattern.compile("do([1-9][0-9]*)d");
private static final Pattern PATTERN_WEEK_OVER_WEEK =
Pattern.compile("wo([1-9][0-9]*)w");
- private static final Pattern PATTERN_MEAN =
Pattern.compile("mean([1-9][0-9]*)w");
- private static final Pattern PATTERN_MEDIAN =
Pattern.compile("median([1-9][0-9]*)w");
- private static final Pattern PATTERN_MIN =
Pattern.compile("min([1-9][0-9]*)w");
- private static final Pattern PATTERN_MAX =
Pattern.compile("max([1-9][0-9]*)w");
+ private static final Pattern PATTERN_MONTH_OVER_MONTH =
Pattern.compile("mo([1-9][0-9]*)m");
+ private static final Pattern PATTERN_MEAN =
Pattern.compile("mean([1-9][0-9]*)(h|d|w|m)");
+ private static final Pattern PATTERN_MEDIAN =
Pattern.compile("median([1-9][0-9]*)(h|d|w|m)");
+ private static final Pattern PATTERN_MIN =
Pattern.compile("min([1-9][0-9]*)(h|d|w|m)");
+ private static final Pattern PATTERN_MAX =
Pattern.compile("max([1-9][0-9]*)(h|d|w|m)");
+
+ private enum Unit {
+ HOUR("h"),
+ DAY("d"),
+ WEEK("w"),
+ MONTH("m");
+
+ final String unit;
+
+ Unit(String unit) {
+ this.unit = unit;
+ }
+
+ static Unit fromUnit(String unit) {
+ for (Unit u : Unit.values()) {
+ if (u.unit.equals(unit)) {
+ return u;
+ }
+ }
+ throw new IllegalArgumentException(String.format("Unknown unit '%s'",
unit));
+ }
+ }
/**
* Returns a configured instance of Baseline for the given, named offset.
The method uses slice and
@@ -43,11 +68,14 @@
* <pre>
* current the time range as specified by start and end)
* none empty time range
+ * hoXh hour-over-hour data points with a lag of X hours)
+ * doXd day-over-day data points with a lag of X days)
* woXw week-over-week data points with a lag of X weeks)
- * meanXw average of data points from the the past X weeks, with a lag
of 1 week)
- * medianXw median of data points from the the past X weeks, with a lag
of 1 week)
- * minXw minimum of data points from the the past X weeks, with a lag
of 1 week)
- * maxXw maximum of data points from the the past X weeks, with a lag
of 1 week)
+ * moXm month-over-month data points with a lag of X months)
+ * meanXU average of data points from the the past X units (hour, day,
month, week), with a lag of 1 unit)
+ * medianXU median of data points from the the past X units (hour, day,
month, week), with a lag of 1 unit)
+ * minXU minimum of data points from the the past X units (hour, day,
month, week), with a lag of 1 unit)
+ * maxXU maximum of data points from the the past X units (hour, day,
month, week), with a lag of 1 unit)
* </pre>
*
* @param offset offset identifier
@@ -74,31 +102,90 @@ public static Baseline parseOffset(String offset, String
timeZoneString) {
return new BaselineNone();
}
+ BaselineAggregateType agg = null;
+ int shift = Integer.MIN_VALUE;
+ int count = Integer.MIN_VALUE;
+ Unit unit = null;
+
+ Matcher mHourOverHour = PATTERN_HOUR_OVER_HOUR.matcher(offset);
+ if (mHourOverHour.find()) {
+ agg = BaselineAggregateType.SUM;
+ shift = Integer.valueOf(mHourOverHour.group(1));
+ count = 1;
+ unit = Unit.HOUR;
+ }
+
+ Matcher mDayOverDay = PATTERN_DAY_OVER_DAY.matcher(offset);
+ if (mDayOverDay.find()) {
+ agg = BaselineAggregateType.SUM;
+ shift = Integer.valueOf(mDayOverDay.group(1));
+ count = 1;
+ unit = Unit.DAY;
+ }
+
Matcher mWeekOverWeek = PATTERN_WEEK_OVER_WEEK.matcher(offset);
if (mWeekOverWeek.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.SUM, 1,
Integer.valueOf(mWeekOverWeek.group(1)), timeZone);
+ agg = BaselineAggregateType.SUM;
+ shift = Integer.valueOf(mWeekOverWeek.group(1));
+ count = 1;
+ unit = Unit.WEEK;
+ }
+
+ Matcher mMonthOverMonth = PATTERN_MONTH_OVER_MONTH.matcher(offset);
+ if (mMonthOverMonth.find()) {
+ agg = BaselineAggregateType.SUM;
+ shift = Integer.valueOf(mMonthOverMonth.group(1));
+ count = 1;
+ unit = Unit.MONTH;
}
Matcher mMean = PATTERN_MEAN.matcher(offset);
if (mMean.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MEAN,
Integer.valueOf(mMean.group(1)), 1, timeZone);
+ agg = BaselineAggregateType.MEAN;
+ shift = 1;
+ count = Integer.valueOf(mMean.group(1));
+ unit = Unit.fromUnit(mMean.group(2));
}
Matcher mMedian = PATTERN_MEDIAN.matcher(offset);
if (mMedian.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MEDIAN,
Integer.valueOf(mMedian.group(1)), 1, timeZone);
+ agg = BaselineAggregateType.MEDIAN;
+ shift = 1;
+ count = Integer.valueOf(mMedian.group(1));
+ unit = Unit.fromUnit(mMedian.group(2));
}
Matcher mMin = PATTERN_MIN.matcher(offset);
if (mMin.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MIN,
Integer.valueOf(mMin.group(1)), 1, timeZone);
+ agg = BaselineAggregateType.MIN;
+ shift = 1;
+ count = Integer.valueOf(mMin.group(1));
+ unit = Unit.fromUnit(mMin.group(2));
}
Matcher mMax = PATTERN_MAX.matcher(offset);
if (mMax.find()) {
- return BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MAX,
Integer.valueOf(mMax.group(1)), 1, timeZone);
+ agg = BaselineAggregateType.MAX;
+ shift = 1;
+ count = Integer.valueOf(mMax.group(1));
+ unit = Unit.fromUnit(mMax.group(2));
+ }
+
+ if (agg == null || unit == null || shift == Integer.MIN_VALUE || count ==
Integer.MIN_VALUE ) {
+ throw new IllegalArgumentException(String.format("Unsupported offset
'%s'", offset));
}
- throw new IllegalArgumentException(String.format("Unknown offset '%s'",
offset));
+ switch (unit) {
+ case HOUR:
+ return BaselineAggregate.fromHourOverHour(agg, count, shift, timeZone);
+ case DAY:
+ return BaselineAggregate.fromDayOverDay(agg, count, shift, timeZone);
+ case WEEK:
+ return BaselineAggregate.fromWeekOverWeek(agg, count, shift, timeZone);
+ case MONTH:
+ return BaselineAggregate.fromMonthOverMonth(agg, count, shift,
timeZone);
+ default:
+ throw new IllegalArgumentException(String.format("Unsupported unit
'%s'", unit));
+ }
}
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/util/DataFrameUtils.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/util/DataFrameUtils.java
index 2d8be7aaf7..36d08f22a1 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/util/DataFrameUtils.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dataframe/util/DataFrameUtils.java
@@ -443,6 +443,13 @@ public static Partial makeOrigin(PeriodType type) {
fields.add(DateTimeFieldType.secondOfMinute());
fields.add(DateTimeFieldType.millisOfSecond());
+ } else if (PeriodType.months().equals(type)) {
+ fields.add(DateTimeFieldType.dayOfMonth());
+ fields.add(DateTimeFieldType.hourOfDay());
+ fields.add(DateTimeFieldType.minuteOfHour());
+ fields.add(DateTimeFieldType.secondOfMinute());
+ fields.add(DateTimeFieldType.millisOfSecond());
+
} else {
throw new IllegalArgumentException(String.format("Unsupported PeriodType
'%s'", type));
}
@@ -450,6 +457,11 @@ public static Partial makeOrigin(PeriodType type) {
int[] zeros = new int[fields.size()];
Arrays.fill(zeros, 0);
+ // workaround for dayOfMonth > 0 constraint
+ if (PeriodType.months().equals(type)) {
+ zeros[0] = 1;
+ }
+
return new Partial(fields.toArray(new DateTimeFieldType[fields.size()]),
zeros);
}
diff --git
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/rootcause/timeseries/BaselineAggregate.java
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/rootcause/timeseries/BaselineAggregate.java
index 4f34804d8e..be929dcd94 100644
---
a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/rootcause/timeseries/BaselineAggregate.java
+++
b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/rootcause/timeseries/BaselineAggregate.java
@@ -231,6 +231,27 @@ public static BaselineAggregate
fromOffsets(BaselineAggregateType type, List<Per
return new BaselineAggregate(type, offsets, timeZone, periodType);
}
+ /**
+ * Returns an instance of BaselineAggregate for the specified type and
{@code numMonths} offsets
+ * computed on a consecutive month-over-month basis starting with a lag of
{@code offsetMonths}.
+ * <br/><b>NOTE:</b> this will apply DST correction
+ *
+ * @see BaselineAggregateType
+ *
+ * @param type aggregation type
+ * @param numMonths number of consecutive months
+ * @param offsetMonths lag for starting consecutive months
+ * @param timeZone time zone
+ * @return BaselineAggregate with given type and monthly offsets
+ */
+ public static BaselineAggregate fromMonthOverMonth(BaselineAggregateType
type, int numMonths, int offsetMonths, DateTimeZone timeZone) {
+ List<Period> offsets = new ArrayList<>();
+ for (int i = 0; i < numMonths; i++) {
+ offsets.add(new Period(0, -1 * (i + offsetMonths), 0, 0, 0, 0, 0, 0,
PeriodType.months()));
+ }
+ return new BaselineAggregate(type, offsets, timeZone, PeriodType.months());
+ }
+
/**
* Returns an instance of BaselineAggregate for the specified type and
{@code numWeeks} offsets
* computed on a consecutive week-over-week basis starting with a lag of
{@code offsetWeeks}.
@@ -384,6 +405,25 @@ public long apply(long... values) {
}
};
+ } else if (PeriodType.months().equals(this.periodType)) {
+ return new Series.LongFunction() {
+ @Override
+ public long apply(long... values) {
+ DateTime dateTime = new DateTime(values[0],
BaselineAggregate.this.timeZone);
+ long months = new Period(origin, dateTime,
BaselineAggregate.this.periodType).getMonths();
+ long days = dateTime.getDayOfMonth() - 1; // workaround for
dayOfMonth > 0 constraint
+ if (days == dateTime.dayOfMonth().getMaximumValue() - 1) {
+ days = 99;
+ }
+
+ long hours = dateTime.getHourOfDay();
+ long minutes = dateTime.getMinuteOfHour();
+ long seconds = dateTime.getSecondOfMinute();
+ long millis = dateTime.getMillisOfSecond();
+ return months * 100000000000L + days * 1000000000L + hours *
10000000L + minutes * 100000L + seconds * 1000L + millis;
+ }
+ };
+
} else {
throw new IllegalArgumentException(String.format("Unsupported PeriodType
'%s'", this.periodType));
}
@@ -469,6 +509,40 @@ public long apply(long... values) {
}
};
+ } else if (PeriodType.months().equals(this.periodType)) {
+ return new Series.LongFunction() {
+ @Override
+ public long apply(long... values) {
+ int months = (int) (values[0] / 100000000000L);
+ int days = (int) ((values[0] / 1000000000L) % 100L);
+ int hours = (int) ((values[0] / 10000000L) % 100L);
+ int minutes = (int) ((values[0] / 100000L) % 100L);
+ int seconds = (int) ((values[0] / 1000L) % 100L);
+ int millis = (int) (values[0] % 1000L);
+
+ DateTime originPlusMonth = origin.plusMonths(months);
+
+ // last day of source month
+ if (days >= 99) {
+ days = originPlusMonth.dayOfMonth().getMaximumValue() - 1;
+ }
+
+ // unsupported destination day (e.g. 31st of Feb)
+ if (originPlusMonth.dayOfMonth().getMaximumValue() <
originPlusMonth.getDayOfMonth() + days) {
+ return LongSeries.NULL;
+ }
+
+ DateTime target = originPlusMonth
+ .plusDays(days)
+ .plusHours(hours)
+ .plusMinutes(minutes)
+ .plusSeconds(seconds)
+ .plusMillis(millis);
+
+ return target.getMillis();
+ }
+ };
+
} else {
throw new IllegalArgumentException(String.format("Unsupported PeriodType
'%s'", this.periodType));
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
