This is an automated email from the ASF dual-hosted git repository. graceguo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push: new 735dcd2 [explore view] add partition as adhoc filter option (#9637) 735dcd2 is described below commit 735dcd20022c90b74cdc58583505a44248c16e57 Author: Grace Guo <grace....@airbnb.com> AuthorDate: Tue Apr 28 23:09:44 2020 -0700 [explore view] add partition as adhoc filter option (#9637) * [explore view] add partition as adhoc option * use adhocFilter Simple Tab * simplify conditional check for custom adhoc filter operator * add simple unit tests --- ...AdhocFilterEditPopoverSimpleTabContent_spec.jsx | 56 +++++++++++++++++++++ superset-frontend/src/explore/AdhocFilter.js | 24 +++++++-- .../explore/components/AdhocFilterEditPopover.jsx | 7 ++- .../AdhocFilterEditPopoverSimpleTabContent.jsx | 58 +++++++++++++++------- .../src/explore/components/AdhocFilterOption.jsx | 2 + .../components/controls/AdhocFilterControl.jsx | 42 ++++++++++++++++ superset-frontend/src/explore/constants.js | 11 ++++ 7 files changed, 176 insertions(+), 24 deletions(-) diff --git a/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx b/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx index d5079ce..1d1f376 100644 --- a/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx +++ b/superset-frontend/spec/javascripts/explore/components/AdhocFilterEditPopoverSimpleTabContent_spec.jsx @@ -52,6 +52,12 @@ const sumValueAdhocMetric = new AdhocMetric({ aggregate: AGGREGATES.SUM, }); +const simpleCustomFilter = new AdhocFilter({ + expressionType: EXPRESSION_TYPES.SIMPLE, + subject: 'ds', + operator: 'LATEST PARTITION', +}); + const options = [ { type: 'VARCHAR(255)', column_name: 'source' }, { type: 'VARCHAR(255)', column_name: 'target' }, @@ -155,6 +161,56 @@ describe('AdhocFilterEditPopoverSimpleTabContent', () => { expect(wrapper.instance().isOperatorRelevant('LIKE')).toBe(false); }); + it('will show LATEST PARTITION operator', () => { + const { wrapper } = setup({ + datasource: { + type: 'table', + datasource_name: 'table1', + schema: 'schema', + }, + adhocFilter: simpleCustomFilter, + partitionColumn: 'ds', + }); + + expect( + wrapper.instance().isOperatorRelevant('LATEST PARTITION', 'ds'), + ).toBe(true); + expect( + wrapper.instance().isOperatorRelevant('LATEST PARTITION', 'value'), + ).toBe(false); + }); + + it('will generate custom sqlExpression for LATEST PARTITION operator', () => { + const testAdhocFilter = new AdhocFilter({ + expressionType: EXPRESSION_TYPES.SIMPLE, + subject: 'ds', + }); + const { wrapper, onChange } = setup({ + datasource: { + type: 'table', + datasource_name: 'table1', + schema: 'schema', + }, + adhocFilter: testAdhocFilter, + partitionColumn: 'ds', + }); + + wrapper.instance().onOperatorChange({ operator: 'LATEST PARTITION' }); + expect( + onChange.lastCall.args[0].equals( + testAdhocFilter.duplicateWith({ + subject: 'ds', + operator: 'LATEST PARTITION', + comparator: null, + clause: 'WHERE', + expressionType: 'SQL', + sqlExpression: + "ds = '{{ presto.latest_partition('schema.table1') }}' ", + }), + ), + ).toBe(true); + }); + it('expands when its multi comparator input field expands', () => { const { wrapper, onHeightChange } = setup(); diff --git a/superset-frontend/src/explore/AdhocFilter.js b/superset-frontend/src/explore/AdhocFilter.js index 7520cec..0c84abc 100644 --- a/superset-frontend/src/explore/AdhocFilter.js +++ b/superset-frontend/src/explore/AdhocFilter.js @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -import { MULTI_OPERATORS } from './constants'; +import { MULTI_OPERATORS, CUSTOM_OPERATORS } from './constants'; export const EXPRESSION_TYPES = { SIMPLE: 'SIMPLE', @@ -41,16 +41,22 @@ const OPERATORS_TO_SQL = { regex: 'regex', 'IS NOT NULL': 'IS NOT NULL', 'IS NULL': 'IS NULL', + 'LATEST PARTITION': ({ datasource }) => { + return `= '{{ presto.latest_partition('${datasource.schema}.${datasource.datasource_name}') }}'`; + }, }; function translateToSql(adhocMetric, { useSimple } = {}) { if (adhocMetric.expressionType === EXPRESSION_TYPES.SIMPLE || useSimple) { const isMulti = MULTI_OPERATORS.indexOf(adhocMetric.operator) >= 0; const subject = adhocMetric.subject; - const operator = OPERATORS_TO_SQL[adhocMetric.operator]; + const operator = + adhocMetric.operator && CUSTOM_OPERATORS.includes(adhocMetric.operator) + ? OPERATORS_TO_SQL[adhocMetric.operator](adhocMetric) + : OPERATORS_TO_SQL[adhocMetric.operator]; const comparator = Array.isArray(adhocMetric.comparator) ? adhocMetric.comparator.join("','") - : adhocMetric.comparator; + : adhocMetric.comparator || ''; return `${subject} ${operator} ${isMulti ? "('" : ''}${comparator}${ isMulti ? "')" : '' }`; @@ -75,8 +81,16 @@ export default class AdhocFilter { ? adhocFilter.sqlExpression : translateToSql(adhocFilter, { useSimple: true }); this.clause = adhocFilter.clause; - this.subject = null; - this.operator = null; + if ( + adhocFilter.operator && + CUSTOM_OPERATORS.includes(adhocFilter.operator) + ) { + this.subject = adhocFilter.subject; + this.operator = adhocFilter.operator; + } else { + this.subject = null; + this.operator = null; + } this.comparator = null; } this.isExtra = !!adhocFilter.isExtra; diff --git a/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx b/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx index 7da098c..4aa19a2 100644 --- a/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx +++ b/superset-frontend/src/explore/components/AdhocFilterEditPopover.jsx @@ -39,6 +39,7 @@ const propTypes = { ]), ).isRequired, datasource: PropTypes.object, + partitionColumn: PropTypes.string, }; const startingWidth = 300; @@ -117,6 +118,7 @@ export default class AdhocFilterEditPopover extends React.Component { onClose, onResize, datasource, + partitionColumn, ...popoverProps } = this.props; @@ -141,9 +143,10 @@ export default class AdhocFilterEditPopover extends React.Component { <AdhocFilterEditPopoverSimpleTabContent adhocFilter={this.state.adhocFilter} onChange={this.onAdhocFilterChange} - options={this.props.options} - datasource={this.props.datasource} + options={options} + datasource={datasource} onHeightChange={this.adjustHeight} + partitionColumn={partitionColumn} /> </Tab> <Tab diff --git a/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx b/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx index 3fb6be6..eadaa54 100644 --- a/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx +++ b/superset-frontend/src/explore/components/AdhocFilterEditPopoverSimpleTabContent.jsx @@ -32,6 +32,8 @@ import { DRUID_ONLY_OPERATORS, HAVING_OPERATORS, MULTI_OPERATORS, + CUSTOM_OPERATORS, + DISABLE_INPUT_OPERATORS, } from '../constants'; import FilterDefinitionOption from './FilterDefinitionOption'; import OnPasteSelect from '../../components/OnPasteSelect'; @@ -50,6 +52,7 @@ const propTypes = { ).isRequired, onHeightChange: PropTypes.func.isRequired, datasource: PropTypes.object, + partitionColumn: PropTypes.string, }; const defaultProps = { @@ -63,6 +66,8 @@ function translateOperator(operator) { return 'not equal to'; } else if (operator === OPERATORS.LIKE) { return 'like'; + } else if (operator === OPERATORS['LATEST PARTITION']) { + return 'use latest_partition template'; } return operator; } @@ -124,10 +129,15 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon subject = option.saved_metric_name || option.label; clause = CLAUSES.HAVING; } + const { operator } = this.props.adhocFilter; this.props.onChange( this.props.adhocFilter.duplicateWith({ subject, clause, + operator: + operator && this.isOperatorRelevant(operator, subject) + ? operator + : null, expressionType: EXPRESSION_TYPES.SIMPLE, }), ); @@ -147,13 +157,26 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon ? currentComparator[0] : currentComparator; } - this.props.onChange( - this.props.adhocFilter.duplicateWith({ - operator: operator && operator.operator, - comparator: newComparator, - expressionType: EXPRESSION_TYPES.SIMPLE, - }), - ); + + if (operator && CUSTOM_OPERATORS.includes(operator.operator)) { + this.props.onChange( + this.props.adhocFilter.duplicateWith({ + subject: this.props.adhocFilter.subject, + clause: CLAUSES.WHERE, + operator: operator && operator.operator, + expressionType: EXPRESSION_TYPES.SQL, + datasource: this.props.datasource, + }), + ); + } else { + this.props.onChange( + this.props.adhocFilter.duplicateWith({ + operator: operator && operator.operator, + comparator: newComparator, + expressionType: EXPRESSION_TYPES.SIMPLE, + }), + ); + } } onInputComparatorChange(event) { @@ -220,7 +243,12 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon } } - isOperatorRelevant(operator) { + isOperatorRelevant(operator, subject) { + if (operator && CUSTOM_OPERATORS.includes(operator)) { + const { partitionColumn } = this.props; + return partitionColumn && subject && subject === partitionColumn; + } + return !( (this.props.datasource.type === 'druid' && TABLE_ONLY_OPERATORS.indexOf(operator) >= 0) || @@ -282,7 +310,9 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon const operatorSelectProps = { placeholder: t('%s operators(s)', Object.keys(OPERATORS).length), options: Object.keys(OPERATORS) - .filter(this.isOperatorRelevant) + .filter(operator => + this.isOperatorRelevant(operator, adhocFilter.subject), + ) .map(operator => ({ operator })), value: adhocFilter.operator, onChange: this.onOperatorChange, @@ -317,10 +347,7 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon showHeader={false} noResultsText={t('type a value here')} refFunc={this.multiComparatorRef} - disabled={ - adhocFilter.operator === 'IS NOT NULL' || - adhocFilter.operator === 'IS NULL' - } + disabled={DISABLE_INPUT_OPERATORS.includes(adhocFilter.operator)} /> ) : ( <input @@ -330,10 +357,7 @@ export default class AdhocFilterEditPopoverSimpleTabContent extends React.Compon value={adhocFilter.comparator || ''} className="form-control input-sm" placeholder={t('Filter value')} - disabled={ - adhocFilter.operator === 'IS NOT NULL' || - adhocFilter.operator === 'IS NULL' - } + disabled={DISABLE_INPUT_OPERATORS.includes(adhocFilter.operator)} /> )} </FormGroup> diff --git a/superset-frontend/src/explore/components/AdhocFilterOption.jsx b/superset-frontend/src/explore/components/AdhocFilterOption.jsx index 10979c2..ea17315 100644 --- a/superset-frontend/src/explore/components/AdhocFilterOption.jsx +++ b/superset-frontend/src/explore/components/AdhocFilterOption.jsx @@ -38,6 +38,7 @@ const propTypes = { ]), ).isRequired, datasource: PropTypes.object, + partitionColumn: PropTypes.string, }; export default class AdhocFilterOption extends React.PureComponent { @@ -80,6 +81,7 @@ export default class AdhocFilterOption extends React.PureComponent { onClose={this.closeFilterEditOverlay} options={this.props.options} datasource={this.props.datasource} + partitionColumn={this.props.partitionColumn} /> ); return ( diff --git a/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx b/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx index 64035b6..79b32ac 100644 --- a/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx +++ b/superset-frontend/src/explore/components/controls/AdhocFilterControl.jsx @@ -21,6 +21,8 @@ import PropTypes from 'prop-types'; import VirtualizedSelect from 'react-virtualized-select'; import { t } from '@superset-ui/translation'; +import { SupersetClient } from '@superset-ui/connection'; + import ControlHeader from '../ControlHeader'; import adhocFilterType from '../../propTypes/adhocFilterType'; import adhocMetricType from '../../propTypes/adhocMetricType'; @@ -90,6 +92,46 @@ export default class AdhocFilterControl extends React.Component { }; } + componentDidMount() { + const { datasource } = this.props; + if (datasource && datasource.type === 'table') { + const dbId = datasource.database ? datasource.database.id : null; + const datasourceName = datasource.datasource_name; + const datasourceSchema = datasource.schema; + + if (dbId && datasourceName && datasourceSchema) { + SupersetClient.get({ + endpoint: `/superset/extra_table_metadata/${dbId}/${datasourceName}/${datasourceSchema}/`, + }).then( + ({ json }) => { + if (json && json.partitions) { + const partitions = json.partitions; + // for now only show latest_partition option + // when table datasource has only 1 partition key. + if ( + partitions && + partitions.cols && + Object.keys(partitions.cols).length === 1 + ) { + const partitionColumn = partitions.cols[0]; + this.valueRenderer = adhocFilter => ( + <AdhocFilterOption + adhocFilter={adhocFilter} + onFilterEdit={this.onFilterEdit} + options={this.state.options} + datasource={this.props.datasource} + partitionColumn={partitionColumn} + /> + ); + } + } + }, + // no error handler, in case of error do not show partition option + ); + } + } + } + UNSAFE_componentWillReceiveProps(nextProps) { if ( this.props.columns !== nextProps.columns || diff --git a/superset-frontend/src/explore/constants.js b/superset-frontend/src/explore/constants.js index 8f1b395..de99199 100644 --- a/superset-frontend/src/explore/constants.js +++ b/superset-frontend/src/explore/constants.js @@ -40,6 +40,7 @@ export const OPERATORS = { regex: 'regex', 'IS NOT NULL': 'IS NOT NULL', 'IS NULL': 'IS NULL', + 'LATEST PARTITION': 'LATEST PARTITION', }; export const TABLE_ONLY_OPERATORS = [OPERATORS.LIKE]; @@ -53,6 +54,16 @@ export const HAVING_OPERATORS = [ OPERATORS['<='], ]; export const MULTI_OPERATORS = [OPERATORS.in, OPERATORS['not in']]; +// CUSTOM_OPERATORS will show operator in simple mode, +// but will generate customized sqlExpression +export const CUSTOM_OPERATORS = [OPERATORS['LATEST PARTITION']]; +// DISABLE_INPUT_OPERATORS will disable filter value input +// in adhocFilter control +export const DISABLE_INPUT_OPERATORS = [ + OPERATORS['IS NOT NULL'], + OPERATORS['IS NULL'], + OPERATORS['LATEST PARTITION'], +]; export const sqlaAutoGeneratedMetricNameRegex = /^(sum|min|max|avg|count|count_distinct)__.*$/i; export const sqlaAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;