This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new acb4416 [deck] allow an array of dynamic of aggregations (#6198)
acb4416 is described below
commit acb44165b406e5a35da9c21c1b6127ed8aca0b7a
Author: Maxime Beauchemin <[email protected]>
AuthorDate: Mon Oct 29 11:08:22 2018 -0700
[deck] allow an array of dynamic of aggregations (#6198)
* [deck] allow an array of dynamic of aggregations
* Adding quantiles
* lint & tests
---
superset/assets/package.json | 1 +
.../visualizations/deckgl/layers/common_spec.jsx | 31 +++++++++++++++++
superset/assets/src/explore/controls.jsx | 23 +++++++++++++
superset/assets/src/explore/visTypes.jsx | 1 +
superset/assets/src/modules/sandbox.js | 2 ++
.../src/visualizations/deckgl/layers/Grid/Grid.jsx | 8 +++--
.../src/visualizations/deckgl/layers/Hex/Hex.jsx | 9 ++---
.../src/visualizations/deckgl/layers/common.jsx | 39 ++++++++++++++++++++--
superset/assets/yarn.lock | 2 +-
9 files changed, 105 insertions(+), 11 deletions(-)
diff --git a/superset/assets/package.json b/superset/assets/package.json
index f52096d..8293e96 100644
--- a/superset/assets/package.json
+++ b/superset/assets/package.json
@@ -62,6 +62,7 @@
"brace": "^0.11.1",
"classnames": "^2.2.5",
"d3": "^3.5.17",
+ "d3-array": "^1.2.4",
"d3-cloud": "^1.2.1",
"d3-color": "^1.2.0",
"d3-hierarchy": "^1.1.5",
diff --git
a/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx
b/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx
new file mode 100644
index 0000000..c11b994
--- /dev/null
+++
b/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx
@@ -0,0 +1,31 @@
+import { max } from 'd3-array';
+import { getAggFunc } from
'../../../../../src/visualizations/deckgl/layers/common';
+
+describe('deckgl layers common', () => {
+ it('getAggFunc', () => {
+ const arr = [10, 0.5, 55, 128, -10];
+ expect(getAggFunc('max')(arr)).toEqual(128);
+ expect(getAggFunc('min')(arr)).toEqual(-10);
+ expect(getAggFunc('count')(arr)).toEqual(5);
+ expect(getAggFunc('median')(arr)).toEqual(10);
+ expect(getAggFunc('mean')(arr)).toEqual(36.7);
+ expect(getAggFunc('p1')(arr)).toEqual(-9.58);
+ expect(getAggFunc('p5')(arr)).toEqual(-7.9);
+ expect(getAggFunc('p95')(arr)).toEqual(113.39999999999998);
+ expect(getAggFunc('p99')(arr)).toEqual(125.08);
+ });
+ it('getAggFunc with accessor', () => {
+ const arr = [{ foo: 1 }, { foo: 2 }, { foo: 3 }];
+ const accessor = o => o.foo;
+ expect(getAggFunc('count')(arr, accessor)).toEqual(3);
+ expect(max(arr, accessor)).toEqual(3);
+ expect(getAggFunc('max', accessor)(arr)).toEqual(3);
+ expect(getAggFunc('min', accessor)(arr)).toEqual(1);
+ expect(getAggFunc('median', accessor)(arr)).toEqual(2);
+ expect(getAggFunc('mean', accessor)(arr)).toEqual(2);
+ expect(getAggFunc('p1', accessor)(arr)).toEqual(1.02);
+ expect(getAggFunc('p5', accessor)(arr)).toEqual(1.1);
+ expect(getAggFunc('p95', accessor)(arr)).toEqual(2.9);
+ expect(getAggFunc('p99', accessor)(arr)).toEqual(2.98);
+ });
+});
diff --git a/superset/assets/src/explore/controls.jsx
b/superset/assets/src/explore/controls.jsx
index d3a25a5..ad32c52 100644
--- a/superset/assets/src/explore/controls.jsx
+++ b/superset/assets/src/explore/controls.jsx
@@ -1317,6 +1317,29 @@ export const controls = {
'computing the total rows and columns'),
},
+ js_agg_function: {
+ type: 'SelectControl',
+ label: t('Dynamic Aggregation Function'),
+ description: t('The function to use when aggregating points into groups'),
+ default: 'sum',
+ clearable: false,
+ renderTrigger: true,
+ choices: formatSelectOptions([
+ 'sum',
+ 'min',
+ 'max',
+ 'mean',
+ 'median',
+ 'count',
+ 'variance',
+ 'deviation',
+ 'p1',
+ 'p5',
+ 'p95',
+ 'p99',
+ ]),
+ },
+
size_from: {
type: 'TextControl',
isInt: true,
diff --git a/superset/assets/src/explore/visTypes.jsx
b/superset/assets/src/explore/visTypes.jsx
index 97ac501..30aba88 100644
--- a/superset/assets/src/explore/visTypes.jsx
+++ b/superset/assets/src/explore/visTypes.jsx
@@ -524,6 +524,7 @@ export const visTypes = {
['mapbox_style', 'viewport'],
['color_picker', 'autozoom'],
['grid_size', 'extruded'],
+ ['js_agg_function', null],
],
},
{
diff --git a/superset/assets/src/modules/sandbox.js
b/superset/assets/src/modules/sandbox.js
index a139013..1d50e15 100644
--- a/superset/assets/src/modules/sandbox.js
+++ b/superset/assets/src/modules/sandbox.js
@@ -1,6 +1,7 @@
// A safe alternative to JS's eval
import vm from 'vm';
import _ from 'underscore';
+import * as d3array from 'd3-array';
import * as colors from './colors';
// Objects exposed here should be treated like a public API
@@ -10,6 +11,7 @@ const GLOBAL_CONTEXT = {
console,
_,
colors,
+ d3array,
};
// Copied/modified from
https://github.com/hacksparrow/safe-eval/blob/master/index.js
diff --git a/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx
b/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx
index c9140ac..b2ccd5b 100644
--- a/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx
+++ b/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx
@@ -1,5 +1,6 @@
import { GridLayer } from 'deck.gl';
-import { commonLayerProps } from '../common';
+
+import { commonLayerProps, getAggFunc } from '../common';
import sandboxedEval from '../../../../modules/sandbox';
import { createDeckGLComponent } from '../../factory';
@@ -17,6 +18,7 @@ export function getLayer(formData, payload, onAddFilter,
setTooltip) {
data = jsFnMutator(data);
}
+ const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight);
return new GridLayer({
id: `grid-layer-${fd.slice_id}`,
data,
@@ -26,8 +28,8 @@ export function getLayer(formData, payload, onAddFilter,
setTooltip) {
extruded: fd.extruded,
maxColor: [c.r, c.g, c.b, 255 * c.a],
outline: false,
- getElevationValue: points => points.reduce((sum, point) => sum +
point.weight, 0),
- getColorValue: points => points.reduce((sum, point) => sum + point.weight,
0),
+ getElevationValue: aggFunc,
+ getColorValue: aggFunc,
...commonLayerProps(fd, setTooltip),
});
}
diff --git a/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx
b/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx
index c3362ed..0e83782 100644
--- a/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx
+++ b/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx
@@ -1,5 +1,6 @@
import { HexagonLayer } from 'deck.gl';
-import { commonLayerProps } from '../common';
+
+import { commonLayerProps, getAggFunc } from '../common';
import sandboxedEval from '../../../../modules/sandbox';
import { createDeckGLComponent } from '../../factory';
@@ -16,7 +17,7 @@ export function getLayer(formData, payload, onAddFilter,
setTooltip) {
const jsFnMutator = sandboxedEval(fd.js_data_mutator);
data = jsFnMutator(data);
}
-
+ const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight);
return new HexagonLayer({
id: `hex-layer-${fd.slice_id}`,
data,
@@ -26,8 +27,8 @@ export function getLayer(formData, payload, onAddFilter,
setTooltip) {
extruded: fd.extruded,
maxColor: [c.r, c.g, c.b, 255 * c.a],
outline: false,
- getElevationValue: points => points.reduce((sum, point) => sum +
point.weight, 0),
- getColorValue: points => points.reduce((sum, point) => sum + point.weight,
0),
+ getElevationValue: aggFunc,
+ getColorValue: aggFunc,
...commonLayerProps(fd, setTooltip),
});
}
diff --git a/superset/assets/src/visualizations/deckgl/layers/common.jsx
b/superset/assets/src/visualizations/deckgl/layers/common.jsx
index a512526..36b05ce 100644
--- a/superset/assets/src/visualizations/deckgl/layers/common.jsx
+++ b/superset/assets/src/visualizations/deckgl/layers/common.jsx
@@ -1,11 +1,12 @@
import React from 'react';
+
import { fitBounds } from 'viewport-mercator-project';
-import d3 from 'd3';
+import * as d3array from 'd3-array';
import sandboxedEval from '../../../modules/sandbox';
export function getBounds(points) {
- const latExt = d3.extent(points, d => d[1]);
- const lngExt = d3.extent(points, d => d[0]);
+ const latExt = d3array.extent(points, d => d[1]);
+ const lngExt = d3array.extent(points, d => d[0]);
return [
[lngExt[0], latExt[0]],
[lngExt[1], latExt[1]],
@@ -73,3 +74,35 @@ export function commonLayerProps(formData, setTooltip,
onSelect) {
pickable: Boolean(onHover),
};
}
+
+const percentiles = {
+ p1: 0.01,
+ p5: 0.05,
+ p95: 0.95,
+ p99: 0.99,
+};
+
+/* Get an a stat function that operates on arrays, aligns with
control=js_agg_function */
+export function getAggFunc(type = 'sum', accessor = null) {
+ if (type === 'count') {
+ return arr => arr.length;
+ }
+ let d3func;
+ if (type in percentiles) {
+ d3func = (arr, acc) => {
+ let sortedArr;
+ if (accessor) {
+ sortedArr = arr.sort((o1, o2) => d3array.ascending(accessor(o1),
accessor(o2)));
+ } else {
+ sortedArr = arr.sort(d3array.ascending);
+ }
+ return d3array.quantile(sortedArr, percentiles[type], acc);
+ };
+ } else {
+ d3func = d3array[type];
+ }
+ if (!accessor) {
+ return arr => d3func(arr);
+ }
+ return arr => d3func(arr.map(accessor));
+}
diff --git a/superset/assets/yarn.lock b/superset/assets/yarn.lock
index 532d9a0..a6ffc95 100644
--- a/superset/assets/yarn.lock
+++ b/superset/assets/yarn.lock
@@ -3394,7 +3394,7 @@ cypress@^3.0.3:
url "0.11.0"
yauzl "2.8.0"
-d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1:
+d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1, d3-array@^1.2.4:
version "1.2.4"
resolved
"https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f";
