This is an automated email from the ASF dual-hosted git repository. bossenti pushed a commit to branch rel/0.95.0 in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 50971d787cc4fb55af8f9aa1f0b613780abfb1f0 Author: Philipp Zehnder <[email protected]> AuthorDate: Tue May 21 15:11:52 2024 +0200 fix(#2855): Issue with deletetransformationrule in adapter (#2887) * fix(#2855): Add e2e test to reproduce the bug * fix(#2855): Fix transformation rules for properties with same prefix --- .../transform/schema/DeleteTransformationRule.java | 2 +- .../schema/DeleteTransformationRuleTest.java | 24 ++++--- .../deleteTransformationRule/nestedInput.json | 6 ++ .../deleteTransformationRule/prefixInput.csv | 2 + .../support/utils/ConnectEventSchemaUtils.ts | 2 +- ui/cypress/support/utils/connect/ConnectUtils.ts | 34 +++++----- .../adapter/rules/deleteTransformationRule.spec.ts | 76 ++++++++++++++++++++++ .../event-schema-preview.component.html | 1 + .../services/transformation-rule.service.spec.ts | 68 ------------------- .../services/transformation-rule.service.ts | 26 +------- 10 files changed, 119 insertions(+), 122 deletions(-) diff --git a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java index 11fb3abe4c..ae2d1c70d9 100644 --- a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java +++ b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRule.java @@ -38,6 +38,6 @@ public class DeleteTransformationRule extends SupportsNestedTransformationRule { @Override protected void applyTransformation(Map<String, Object> event, List<String> eventKeys) { - event.remove(eventKeys.get(0)); + event.remove(eventKeys.get(0)); } } diff --git a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java index 0579efb627..7253b33bd5 100644 --- a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java +++ b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/schema/DeleteTransformationRuleTest.java @@ -18,41 +18,39 @@ package org.apache.streampipes.connect.shared.preprocessing.transform.schema; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.HashMap; import java.util.List; -import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; public class DeleteTransformationRuleTest { @Test public void transformSimple() { - Map<String, Object> event = new HashMap<>(); + var event = new HashMap<String, Object>(); event.put("key", "value"); - DeleteTransformationRule deleteRule = new DeleteTransformationRule(List.of("key")); + var deleteRule = new DeleteTransformationRule(List.of("key")); - Map<String, Object> result = deleteRule.apply(event); + var result = deleteRule.apply(event); - Assertions.assertEquals(0, - result.keySet().size()); + assertEquals(0, result.keySet().size()); } @Test public void transformNested() { - Map<String, Object> child = new HashMap<>(); + var child = new HashMap<String, Object>(); child.put("child", "value"); - Map<String, Object> event = new HashMap<>(); + var event = new HashMap<String, Object>(); event.put("parent", child); - DeleteTransformationRule deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child")); + var deleteRule = new DeleteTransformationRule(Arrays.asList("parent", "child")); - Map<String, Object> result = deleteRule.apply(event); + var result = deleteRule.apply(event); - Assertions.assertEquals(1, - result.keySet().size()); + assertEquals(1, result.keySet().size()); } } diff --git a/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json b/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json new file mode 100644 index 0000000000..92177c23ed --- /dev/null +++ b/ui/cypress/fixtures/connect/deleteTransformationRule/nestedInput.json @@ -0,0 +1,6 @@ +{ + "timestamp": 1667904471000, + "parent": { + "child": "text" + } +} diff --git a/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv b/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv new file mode 100644 index 0000000000..5280fe3838 --- /dev/null +++ b/ui/cypress/fixtures/connect/deleteTransformationRule/prefixInput.csv @@ -0,0 +1,2 @@ +timestamp;reserved bit;reserved bit_1;reserved bit_2 +1715356080000;true;true;true diff --git a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts b/ui/cypress/support/utils/ConnectEventSchemaUtils.ts index 1d933399ff..495a47c738 100644 --- a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts +++ b/ui/cypress/support/utils/ConnectEventSchemaUtils.ts @@ -178,7 +178,7 @@ export class ConnectEventSchemaUtils { } public static deleteProperty(propertyName: string) { - cy.dataCy('delete-property-' + propertyName, { timeout: 10000 }) + cy.dataCy('"delete-property-' + propertyName + '"', { timeout: 10000 }) .children() .click({ force: true }); cy.dataCy('connect-schema-delete-properties-btn', { diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts index cfe06bfc6a..ed3b7bcb5b 100644 --- a/ui/cypress/support/utils/connect/ConnectUtils.ts +++ b/ui/cypress/support/utils/connect/ConnectUtils.ts @@ -276,29 +276,33 @@ export class ConnectUtils { public static setUpPreprocessingRuleTest( overwriteTimestamp: boolean, + adapterConfigurationBuilder?: AdapterBuilder, ): AdapterInput { - const adapterConfiguration = AdapterBuilder.create('File_Stream') - .setStoreInDataLake() - .setTimestampProperty('timestamp') - .addProtocolInput( - 'radio', - 'speed', - 'fastest_\\(ignore_original_time\\)', - ) - .addProtocolInput('radio', 'replayonce', 'yes') - .setName('Adapter to test rules') - .setFormat('csv') - .addFormatInput('input', ConnectBtns.csvDelimiter(), ';') - .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check'); + if (!adapterConfigurationBuilder) { + adapterConfigurationBuilder = AdapterBuilder.create('File_Stream') + .setStoreInDataLake() + .setTimestampProperty('timestamp') + .addProtocolInput( + 'radio', + 'speed', + 'fastest_\\(ignore_original_time\\)', + ) + .addProtocolInput('radio', 'replayonce', 'yes') + .setName('Adapter to test rules') + .setFormat('csv') + .addFormatInput('input', ConnectBtns.csvDelimiter(), ';') + .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check'); + } if (overwriteTimestamp) { - adapterConfiguration.addProtocolInput( + adapterConfigurationBuilder.addProtocolInput( 'checkbox', 'replaceTimestamp', 'check', ); } - adapterConfiguration = adapterConfiguration.build(); + + const adapterConfiguration = adapterConfigurationBuilder.build(); ConnectUtils.goToConnect(); ConnectUtils.goToNewAdapterPage(); diff --git a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts new file mode 100644 index 0000000000..debbc5473f --- /dev/null +++ b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils'; +import { FileManagementUtils } from '../../../support/utils/FileManagementUtils'; +import { ConnectEventSchemaUtils } from '../../../support/utils/ConnectEventSchemaUtils'; +import { AdapterBuilder } from '../../../support/builder/AdapterBuilder'; + +describe('Connect delete rule transformation', () => { + beforeEach('Setup Test', () => { + cy.initStreamPipesTest(); + }); + + it('Test delete with same prefix', () => { + FileManagementUtils.addFile( + 'connect/deleteTransformationRule/prefixInput.csv', + ); + + ConnectUtils.setUpPreprocessingRuleTest(false); + + ConnectEventSchemaUtils.deleteProperty('reserved bit'); + ConnectEventSchemaUtils.deleteProperty('reserved bit_1'); + ConnectEventSchemaUtils.deleteProperty('reserved bit_2'); + + cy.dataCy('schema-preview-result-event').should( + 'have.text', + '{\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01715356080000}', + ); + }); + + it('Test delete nested properties', () => { + FileManagementUtils.addFile( + 'connect/deleteTransformationRule/nestedInput.json', + ); + + const adapterConfigurationBuilder = AdapterBuilder.create('File_Stream') + .setStoreInDataLake() + .setTimestampProperty('timestamp') + .addProtocolInput( + 'radio', + 'speed', + 'fastest_\\(ignore_original_time\\)', + ) + .addProtocolInput('radio', 'replayonce', 'yes') + .setName('Adapter to test rules') + .setFormat('json'); + + ConnectUtils.setUpPreprocessingRuleTest( + false, + adapterConfigurationBuilder, + ); + + // Test to delete the child property + ConnectEventSchemaUtils.deleteProperty('child'); + + // The resulting string contains non-breaking spaces character (\u00A0) + cy.dataCy('schema-preview-result-event').should( + 'have.text', + '{\u00A0\u00A0\u00A0\u00A0"parent":\u00A0{},\u00A0\u00A0\u00A0\u00A0"timestamp":\u00A01667904471000}', + ); + }); +}); diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html index 41d8e5446c..41ae59c67b 100644 --- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html +++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html @@ -49,6 +49,7 @@ <pre [innerHTML]="desiredField | jsonpretty" class="preview-text" + data-cy="schema-preview-result-event" ></pre> </sp-basic-inner-panel> </div> diff --git a/ui/src/app/connect/services/transformation-rule.service.spec.ts b/ui/src/app/connect/services/transformation-rule.service.spec.ts index 39c938b913..a0030bf34f 100644 --- a/ui/src/app/connect/services/transformation-rule.service.spec.ts +++ b/ui/src/app/connect/services/transformation-rule.service.spec.ts @@ -19,7 +19,6 @@ import { TransformationRuleService } from './transformation-rule.service'; import { CreateNestedRuleDescription, - DeleteRuleDescription, EventPropertyNested, EventPropertyPrimitive, EventPropertyUnion, @@ -196,73 +195,6 @@ describe('TransformationRuleService', () => { expect(result.length).toBe(1); expect(result[0].oldRuntimeKey).toBe('a'); - // expect(result[0].newRuntimeKey).toBe('b.a'); - }); - - it('Delete simple', () => { - const oldEventSchema: EventSchema = new EventSchema(); - const eventProperty: EventPropertyPrimitive = - new EventPropertyPrimitive(); - eventProperty.runtimeName = 'a'; - oldEventSchema.eventProperties = []; - oldEventSchema.eventProperties.push(eventProperty); - - const newEventSchema: EventSchema = new EventSchema(); - newEventSchema.eventProperties = []; - - const result: DeleteRuleDescription[] = service.getDeleteRules( - newEventSchema.eventProperties, - oldEventSchema, - newEventSchema, - ); - - expect(result.length).toBe(1); - expect(result[0].runtimeKey).toBe('a'); - }); - - it('Delete nested', () => { - const oldEventSchema: EventSchema = new EventSchema(); - const eventProperty: EventPropertyPrimitive = - new EventPropertyPrimitive(); - eventProperty.elementId = 'id_2'; - eventProperty.runtimeName = 'a'; - const eventPropertyNested: EventPropertyNested = - new EventPropertyNested(); - eventPropertyNested.elementId = 'id_1'; - eventPropertyNested.eventProperties = []; - eventPropertyNested.eventProperties.push(eventProperty); - eventPropertyNested.runtimeName = 'b'; - oldEventSchema.eventProperties = []; - oldEventSchema.eventProperties.push(eventPropertyNested); - - let newEventSchema: EventSchema = new EventSchema(); - const newEventPropertyNested: EventPropertyNested = - new EventPropertyNested(); - newEventPropertyNested.elementId = 'id_1'; - newEventPropertyNested.runtimeName = 'b'; - newEventPropertyNested.eventProperties = []; - newEventSchema.eventProperties = []; - newEventSchema.eventProperties.push(newEventPropertyNested); - - let result: DeleteRuleDescription[] = service.getDeleteRules( - newEventSchema.eventProperties, - oldEventSchema, - newEventSchema, - ); - - expect(result.length).toBe(1); - expect(result[0].runtimeKey).toBe('b.a'); - - newEventSchema = new EventSchema(); - newEventSchema.eventProperties = []; - result = service.getDeleteRules( - newEventSchema.eventProperties, - oldEventSchema, - newEventSchema, - ); - - expect(result.length).toBe(1); - expect(result[0].runtimeKey).toBe('b'); }); it('Rename simple', () => { diff --git a/ui/src/app/connect/services/transformation-rule.service.ts b/ui/src/app/connect/services/transformation-rule.service.ts index f1cca8565c..b0fd79fa3d 100644 --- a/ui/src/app/connect/services/transformation-rule.service.ts +++ b/ui/src/app/connect/services/transformation-rule.service.ts @@ -125,13 +125,7 @@ export class TransformationRuleService { targetSchema, ), ) - .concat( - this.getDeleteRules( - targetSchema.eventProperties, - originalSchema, - targetSchema, - ), - ) + .concat(this.getDeleteRules(originalSchema, targetSchema)) .concat( this.getUnitTransformRules( targetSchema.eventProperties, @@ -299,11 +293,10 @@ export class TransformationRuleService { } public getDeleteRules( - newEventProperties: EventProperty[], oldEventSchema: EventSchema, newEventSchema: EventSchema, ): DeleteRuleDescription[] { - let resultKeys: string[] = []; + const resultKeys: string[] = []; const allNewIds: string[] = this.getAllIds( newEventSchema.eventProperties, @@ -323,21 +316,6 @@ export class TransformationRuleService { } } - const uniqEs6 = arrArg => { - return arrArg.filter((elem, pos, arr) => { - let r = true; - for (const a of arr) { - if (elem.startsWith(a) && a !== elem) { - r = false; - } - } - - return r; - }); - }; - - resultKeys = uniqEs6(resultKeys); - const resultRules: DeleteRuleDescription[] = []; for (const key of resultKeys) { const rule: DeleteRuleDescription = new DeleteRuleDescription();
