This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch
3430-filtering-for-numerical-dimension-properties-in-data-explorer
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/3430-filtering-for-numerical-dimension-properties-in-data-explorer
by this push:
new eead931ee7 fix(#3430): Numerical values are now handled as expected by
filters
eead931ee7 is described below
commit eead931ee7365006e8d071fae01cff8c60d2d609
Author: Philipp Zehnder <[email protected]>
AuthorDate: Thu Jan 23 09:48:43 2025 +0100
fix(#3430): Numerical values are now handled as expected by filters
---
.../param/model/WhereClauseParams.java | 46 +++++++++++++++-------
.../datalake/filterDimensionProperties.csv | 3 --
.../datalake/filterNumericalStringProperties.csv | 3 ++
....ts => filterNumericalStringProperties.spec.ts} | 21 +++++++---
.../escape-number-filter.service.ts | 45 +++++++++++++++++++++
...nel-row-value-input-autocomplete.component.html | 10 ++---
...panel-row-value-input-autocomplete.component.ts | 22 ++++++++++-
...-selection-panel-row-value-input.component.html | 3 +-
...er-selection-panel-row-value-input.component.ts | 24 ++++++++++-
9 files changed, 143 insertions(+), 34 deletions(-)
diff --git
a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java
b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java
index 6bf5589d99..01613487a5 100644
---
a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java
+++
b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java
@@ -34,17 +34,21 @@ public class WhereClauseParams implements IQueryStatement {
private final List<FilterCondition> filterConditions;
- private WhereClauseParams(Long startTime,
- Long endTime,
- String whereConditions) {
+ private WhereClauseParams(
+ Long startTime,
+ Long endTime,
+ String whereConditions
+ ) {
this(startTime, endTime);
if (whereConditions != null) {
buildConditions(whereConditions);
}
}
- private WhereClauseParams(Long startTime,
- Long endTime) {
+ private WhereClauseParams(
+ Long startTime,
+ Long endTime
+ ) {
this.filterConditions = new ArrayList<>();
this.buildTimeConditions(startTime, endTime);
}
@@ -56,8 +60,10 @@ public class WhereClauseParams implements IQueryStatement {
}
}
- public static WhereClauseParams from(Long startTime,
- Long endTime) {
+ public static WhereClauseParams from(
+ Long startTime,
+ Long endTime
+ ) {
return new WhereClauseParams(startTime, endTime);
}
@@ -65,14 +71,18 @@ public class WhereClauseParams implements IQueryStatement {
return new WhereClauseParams(whereConditions);
}
- public static WhereClauseParams from(Long startTime,
- Long endTime,
- String whereConditions) {
+ public static WhereClauseParams from(
+ Long startTime,
+ Long endTime,
+ String whereConditions
+ ) {
return new WhereClauseParams(startTime, endTime, whereConditions);
}
- private void buildTimeConditions(Long startTime,
- Long endTime) {
+ private void buildTimeConditions(
+ Long startTime,
+ Long endTime
+ ) {
if (startTime == null) {
this.filterConditions.add(buildTimeBoundary(endTime, LT));
} else if (endTime == null) {
@@ -99,7 +109,7 @@ public class WhereClauseParams implements IQueryStatement {
private Object returnCondition(String inputCondition) {
if (isQuotedString(inputCondition)) {
- return inputCondition.substring(1, inputCondition.length() - 1);
+ return removeQuotes(inputCondition);
} else if (NumberUtils.isParsable(inputCondition)) {
return Double.parseDouble(inputCondition);
} else if (isBoolean(inputCondition)) {
@@ -110,7 +120,15 @@ public class WhereClauseParams implements IQueryStatement {
}
private boolean isQuotedString(String input) {
- return input.startsWith("\"") && input.endsWith("\"");
+ if (input.startsWith("\"") && input.endsWith("\"")) {
+ String content = removeQuotes(input);
+ return NumberUtils.isParsable(content);
+ }
+ return false;
+ }
+
+ private String removeQuotes(String input) {
+ return input.substring(1, input.length() - 1);
}
private boolean isBoolean(String input) {
diff --git a/ui/cypress/fixtures/datalake/filterDimensionProperties.csv
b/ui/cypress/fixtures/datalake/filterDimensionProperties.csv
deleted file mode 100644
index 7b42dc4df1..0000000000
--- a/ui/cypress/fixtures/datalake/filterDimensionProperties.csv
+++ /dev/null
@@ -1,3 +0,0 @@
-timestamp;dimensionKey;value
-1737123058000;1.0;10.0
-1737123059000;2.0;10.0
diff --git a/ui/cypress/fixtures/datalake/filterNumericalStringProperties.csv
b/ui/cypress/fixtures/datalake/filterNumericalStringProperties.csv
new file mode 100644
index 0000000000..3cf0902c26
--- /dev/null
+++ b/ui/cypress/fixtures/datalake/filterNumericalStringProperties.csv
@@ -0,0 +1,3 @@
+timestamp;dimensionKey;v1
+1737123058000;1.0;a
+1737123059000;2.0;20
diff --git a/ui/cypress/tests/datalake/filterDimensionProperties.spec.ts
b/ui/cypress/tests/datalake/filterNumericalStringProperties.spec.ts
similarity index 85%
rename from ui/cypress/tests/datalake/filterDimensionProperties.spec.ts
rename to ui/cypress/tests/datalake/filterNumericalStringProperties.spec.ts
index 9e1bb1d125..003e5c23c0 100644
--- a/ui/cypress/tests/datalake/filterDimensionProperties.spec.ts
+++ b/ui/cypress/tests/datalake/filterNumericalStringProperties.spec.ts
@@ -28,7 +28,9 @@ describe('Validate that filter works for numerical dimension
property', () => {
beforeEach('Setup Test', () => {
cy.initStreamPipesTest();
- FileManagementUtils.addFile('datalake/filterDimensionProperties.csv');
+ FileManagementUtils.addFile(
+ 'datalake/filterNumericalStringProperties.csv',
+ );
const adapterInput = AdapterBuilder.create('File_Stream')
.setName('Test Adapter')
.setTimestampProperty('timestamp')
@@ -61,11 +63,18 @@ describe('Validate that filter works for numerical
dimension property', () => {
// select filter for tag
DataLakeUtils.selectDataConfig();
- const filterConfig = new DataLakeFilterConfig(
- 'dimensionKey',
- '1.0',
- '=',
- );
+ var filterConfig = new DataLakeFilterConfig('dimensionKey', '1.0',
'=');
+ DataLakeUtils.dataConfigAddFilter(filterConfig);
+
+ // validate data in table is filtered
+ DataLakeWidgetTableUtils.checkAmountOfRows(1);
+
+ // remove filter
+ DataLakeUtils.dataConfigRemoveFilter();
+
+ DataLakeUtils.selectDataConfig();
+
+ filterConfig = new DataLakeFilterConfig('v1', '20', '=');
DataLakeUtils.dataConfigAddFilter(filterConfig);
// validate data in table is filtered
diff --git
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/escape-number-filter.service.ts
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/escape-number-filter.service.ts
new file mode 100644
index 0000000000..c8d7dc5534
--- /dev/null
+++
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/escape-number-filter.service.ts
@@ -0,0 +1,45 @@
+import { Injectable } from '@angular/core';
+
+@Injectable({
+ providedIn: 'root',
+})
+export class EscapeNumberFilterService {
+ // Method to remove enclosing double quotes
+ removeEnclosingQuotes(value: string): string {
+ return value?.replace(/^"|"$/g, '');
+ }
+
+ // Updates the filter value based on the field type and input value.
+ // Ensures that numeric values are wrapped in double quotes to prevent
parsing issues on the backend.
+ // This check is necessary because the filter value is transmitted as a
triple [field, operator, value],
+ // which causes the type information to be lost. Once the API is changed
to retain type information,
+ // this service can be removed.
+ escapeIfNumberValue(
+ filter: any,
+ value: string,
+ tagValues: Map<string, string[]>,
+ ): string {
+ const isTagValueKey = this.checkIfFilterOnTagValue(filter, tagValues);
+ const isNumericValue = this.checkIfNumericalValue(value);
+
+ if (isNumericValue && (isTagValueKey || !filter?.field?.numeric)) {
+ return `"${value}"`;
+ } else {
+ return value;
+ }
+ }
+
+ private checkIfFilterOnTagValue(
+ filter: any,
+ tagValues: Map<string, string[]>,
+ ): boolean {
+ return (
+ tagValues.has(filter?.field?.runtimeName) &&
+ tagValues.get(filter.field.runtimeName)?.length > 0
+ );
+ }
+
+ private checkIfNumericalValue(value: string): boolean {
+ return !isNaN(Number(value));
+ }
+}
diff --git
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input-autocomplete/filter-selection-panel-row-value-input-autocomplete.component.html
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input-autocomplete/filter-selection-panel-row-value-inpu
[...]
index 9ff3259aca..81ff27305b 100644
---
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input-autocomplete/filter-selection-panel-row-value-input-autocomplete.component.html
+++
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input-autocomplete/filter-selection-panel-row-value-input-autocomplete.component.html
@@ -21,7 +21,7 @@
placeholder="Value"
aria-label="Number"
matInput
- [(ngModel)]="filter.value"
+ [(ngModel)]="value"
(change)="updateParentComponent()"
data-cy="design-panel-data-settings-filter-value"
[matAutocomplete]="auto"
@@ -30,12 +30,12 @@
#auto="matAutocomplete"
(optionSelected)="updateParentComponent()"
>
- @for (value of tagValues.get(filter.field.runtimeName); track value) {
+ @for (option of tagValues.get(filter.field.runtimeName); track option)
{
<mat-option
- [attr.data-cy]="'autocomplete-value-' + value"
- [value]="value"
+ [attr.data-cy]="'autocomplete-value-' + option"
+ [value]="option"
>
- {{ value }}
+ {{ option }}
</mat-option>
}
</mat-autocomplete>
diff --git
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input-autocomplete/filter-selection-panel-row-value-input-autocomplete.component.ts
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input-autocomplete/filter-selection-panel-row-value-input-
[...]
index e02ca4b998..da0c7a9f35 100644
---
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input-autocomplete/filter-selection-panel-row-value-input-autocomplete.component.ts
+++
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input-autocomplete/filter-selection-panel-row-value-input-autocomplete.component.ts
@@ -15,15 +15,18 @@
* limitations under the License.
*
*/
-import { Component, EventEmitter, Input, Output } from '@angular/core';
+import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
import { SelectedFilter } from '@streampipes/platform-services';
+import { EscapeNumberFilterService } from '../escape-number-filter.service';
@Component({
selector: 'sp-filter-selection-panel-row-value-input-autocomplete',
templateUrl:
'./filter-selection-panel-row-value-input-autocomplete.component.html',
})
-export class FilterSelectionPanelRowValueInputAutocompleteComponent {
+export class FilterSelectionPanelRowValueInputAutocompleteComponent
+ implements OnInit
+{
@Input()
public filter: SelectedFilter;
@@ -33,7 +36,22 @@ export class
FilterSelectionPanelRowValueInputAutocompleteComponent {
@Output()
public update = new EventEmitter<void>();
+ public value: string;
+
+ constructor(private escapeNumberFilterService: EscapeNumberFilterService)
{}
+
+ ngOnInit(): void {
+ this.value = this.escapeNumberFilterService.removeEnclosingQuotes(
+ this.filter.value,
+ );
+ }
+
updateParentComponent() {
+ this.filter.value = this.escapeNumberFilterService.escapeIfNumberValue(
+ this.filter,
+ this.value,
+ this.tagValues,
+ );
this.update.emit();
}
}
diff --git
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input/filter-selection-panel-row-value-input.component.html
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input/filter-selection-panel-row-value-input.component.html
index 38614ea5b1..8f7c507b79 100644
---
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input/filter-selection-panel-row-value-input.component.html
+++
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input/filter-selection-panel-row-value-input.component.html
@@ -16,13 +16,12 @@
~
-->
<mat-form-field color="accent" appearance="outline" class="w-140-px mr-5">
- <mat-label>Value</mat-label>
<input
type="text"
placeholder="Value"
aria-label="Number"
matInput
- [(ngModel)]="filter.value"
+ [(ngModel)]="value"
(change)="updateParentComponent()"
data-cy="design-panel-data-settings-filter-value"
/>
diff --git
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input/filter-selection-panel-row-value-input.component.ts
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input/filter-selection-panel-row-value-input.component.ts
index 4e52a4bf97..2d6031672b 100644
---
a/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input/filter-selection-panel-row-value-input.component.ts
+++
b/ui/src/app/data-explorer/components/data-view/data-view-designer-panel/data-settings/filter-selection-panel/filter-selection-panel-row/filter-selection-panel-row-value-input/filter-selection-panel-row-value-input.component.ts
@@ -15,21 +15,41 @@
* limitations under the License.
*
*/
-import { Component, EventEmitter, Input, Output } from '@angular/core';
+import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
import { SelectedFilter } from '@streampipes/platform-services';
+import { EscapeNumberFilterService } from '../escape-number-filter.service';
@Component({
selector: 'sp-filter-selection-panel-row-value-input',
templateUrl: './filter-selection-panel-row-value-input.component.html',
})
-export class FilterSelectionPanelRowValueInputComponent {
+export class FilterSelectionPanelRowValueInputComponent implements OnInit {
@Input()
public filter: SelectedFilter;
+ // This is only required to correctly escape numbers
+ @Input()
+ public tagValues: Map<string, string[]>;
+
@Output()
public update = new EventEmitter<void>();
+ public value: string;
+
+ constructor(private escapeNumberFilterService: EscapeNumberFilterService)
{}
+
+ ngOnInit(): void {
+ this.value = this.escapeNumberFilterService.removeEnclosingQuotes(
+ this.filter.value,
+ );
+ }
+
updateParentComponent() {
+ this.filter.value = this.escapeNumberFilterService.escapeIfNumberValue(
+ this.filter,
+ this.value,
+ this.tagValues,
+ );
this.update.emit();
}
}
