This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch
2954-data-type-change-in-adapter-causes-data-storage-failure-in-influxdb
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/2954-data-type-change-in-adapter-causes-data-storage-failure-in-influxdb
by this push:
new ecf420aa3b fix(#2954): Add test to reproduce problem
ecf420aa3b is described below
commit ecf420aa3b05a197383cc070fa1f73867bbcb437
Author: Philipp Zehnder <[email protected]>
AuthorDate: Wed Jun 26 08:53:28 2024 +0200
fix(#2954): Add test to reproduce problem
---
ui/cypress/support/utils/connect/ConnectBtns.ts | 13 +++++++
.../utils/{ => connect}/ConnectEventSchemaUtils.ts | 30 ++++++++-------
ui/cypress/support/utils/connect/ConnectUtils.ts | 10 ++++-
ui/cypress/support/utils/datalake/DataLakeBtns.ts | 24 ++++++++++++
ui/cypress/support/utils/datalake/DataLakeUtils.ts | 15 ++++++++
ui/cypress/tests/adapter/editAdapter.smoke.spec.ts | 45 +++++++++++++++++++++-
.../adapter/editAdapterSettingsAndPipeline.spec.ts | 2 +-
.../adapter/editAdapterValuesAndFields.spec.ts | 6 +--
.../adapter/rules/deleteTransformationRule.spec.ts | 2 +-
.../tests/adapter/rules/schemaRules.smoke.spec.ts | 2 +-
ui/cypress/tests/adapter/rules/streamRules.spec.ts | 2 +-
ui/cypress/tests/adapter/rules/valueRules.ts | 2 +-
.../datalake-configuration.component.html | 4 +-
13 files changed, 132 insertions(+), 25 deletions(-)
diff --git a/ui/cypress/support/utils/connect/ConnectBtns.ts
b/ui/cypress/support/utils/connect/ConnectBtns.ts
index 0fab8973c0..5b18603a2e 100644
--- a/ui/cypress/support/utils/connect/ConnectBtns.ts
+++ b/ui/cypress/support/utils/connect/ConnectBtns.ts
@@ -40,6 +40,19 @@ export class ConnectBtns {
return cy.dataCy('store-edit-adapter');
}
+ public static changeRuntimeType() {
+ return cy.dataCy('connect-change-runtime-type', { timeout: 10000 });
+ }
+
+ public static updateAndMigratePipelines() {
+ return cy.dataCy('btn-update-adapter-migrate-pipelines');
+ }
+
+ public static nextBtn() {
+ return cy.get('button').contains('Next').parent();
+ }
+
+
// ===================== Adapter settings btns ==========================
public static adapterSettingsStartAdapter() {
return cy.dataCy('adapter-settings-start-adapter-btn');
diff --git a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts
b/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
similarity index 88%
rename from ui/cypress/support/utils/ConnectEventSchemaUtils.ts
rename to ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
index 663a42edbf..3d84b50380 100644
--- a/ui/cypress/support/utils/ConnectEventSchemaUtils.ts
+++ b/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
@@ -1,21 +1,23 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*
*/
+import { ConnectBtns } from './ConnectBtns';
+
export class ConnectEventSchemaUtils {
public static markPropertyAsDimension(propertyName: string) {
cy.dataCy('property-scope-' + propertyName, { timeout: 10000 })
@@ -198,7 +200,7 @@ export class ConnectEventSchemaUtils {
dataType: string,
) {
ConnectEventSchemaUtils.clickEditProperty(propertyName);
- cy.dataCy('connect-change-runtime-type')
+ ConnectBtns.changeRuntimeType()
.click()
.get('mat-option')
.contains(dataType)
@@ -208,7 +210,7 @@ export class ConnectEventSchemaUtils {
cy.dataCy('edit-' + propertyName, { timeout: 10000 }).click({
force: true,
});
- cy.dataCy('connect-change-runtime-type', { timeout: 10000 }).contains(
+ ConnectBtns.changeRuntimeType().contains(
dataType,
);
cy.wait(1000);
diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts
b/ui/cypress/support/utils/connect/ConnectUtils.ts
index 6fe6cfd708..fa30cd01e0 100644
--- a/ui/cypress/support/utils/connect/ConnectUtils.ts
+++ b/ui/cypress/support/utils/connect/ConnectUtils.ts
@@ -18,7 +18,7 @@
import { StaticPropertyUtils } from '../StaticPropertyUtils';
import { AdapterInput } from '../../model/AdapterInput';
-import { ConnectEventSchemaUtils } from '../ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from './ConnectEventSchemaUtils';
import { DataLakeUtils } from '../datalake/DataLakeUtils';
import { ConnectBtns } from './ConnectBtns';
import { AdapterBuilder } from '../../builder/AdapterBuilder';
@@ -208,6 +208,14 @@ export class ConnectUtils {
);
}
+ public static storeAndStartEditedAdapter() {
+ ConnectUtils.finishEventSchemaConfiguration();
+ ConnectBtns.storeEditAdapter().click();
+ ConnectBtns.updateAndMigratePipelines().click();
+ ConnectUtils.closeAdapterPreview();
+ ConnectBtns.startAdapter().click();
+ }
+
public static deleteAdapterAndAssociatedPipelines(switchUserCheck = false)
{
// Delete adapter and associated pipelines
this.goToConnect();
diff --git a/ui/cypress/support/utils/datalake/DataLakeBtns.ts
b/ui/cypress/support/utils/datalake/DataLakeBtns.ts
new file mode 100644
index 0000000000..03ab7e1759
--- /dev/null
+++ b/ui/cypress/support/utils/datalake/DataLakeBtns.ts
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+export class DataLakeBtns {
+
+ public static refreshDataLakeMeasures() {
+ return cy.dataCy('refresh-data-lake-measures');
+ }
+ }
diff --git a/ui/cypress/support/utils/datalake/DataLakeUtils.ts
b/ui/cypress/support/utils/datalake/DataLakeUtils.ts
index 1788212551..064a73b494 100644
--- a/ui/cypress/support/utils/datalake/DataLakeUtils.ts
+++ b/ui/cypress/support/utils/datalake/DataLakeUtils.ts
@@ -332,4 +332,19 @@ export class DataLakeUtils {
return currentDate;
}
+
+
+ public static waitForCountingResults() {
+ cy.dataCy('datalake-number-of-events-spinner', { timeout: 10000 })
+ .should('exist');
+ cy.dataCy('datalake-number-of-events-spinner', { timeout: 10000 })
+ .should('not.exist');
+ }
+
+ public static getDatalakeNumberOfEvents(): Cypress.Chainable<string> {
+ return cy.dataCy('datalake-number-of-events', { timeout: 10000 })
+ .should('be.visible')
+ .invoke('text')
+ .then((text) => text.trim());
+ }
}
diff --git a/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
b/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
index ba8a1f54e0..b346273e24 100644
--- a/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
+++ b/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
@@ -19,15 +19,18 @@
import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
import { ConnectBtns } from '../../support/utils/connect/ConnectBtns';
import { AdapterBuilder } from '../../support/builder/AdapterBuilder';
+import { ConnectEventSchemaUtils } from
'../../support/utils/connect/ConnectEventSchemaUtils';
+import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
+import { DataLakeBtns } from '../../support/utils/datalake/DataLakeBtns';
describe('Test Edit Adapter', () => {
beforeEach('Setup Test', () => {
// To set up test add a stream adapter that can be configured
cy.initStreamPipesTest();
- ConnectUtils.addMachineDataSimulator('simulator');
});
it('Successfully edit adapter', () => {
+ ConnectUtils.addMachineDataSimulator('simulator');
const newAdapterName = 'Edited Adapter';
ConnectUtils.goToConnect();
@@ -68,4 +71,44 @@ describe('Test Edit Adapter', () => {
cy.dataCy('adapter-name').contains(newAdapterName);
cy.get('.sp-dialog-content').contains(newAdapterName);
});
+
+ it('Successfully edit adapter with persistence pipeline', () => {
+ ConnectUtils.addMachineDataSimulator('simulator', true);
+
+ ConnectUtils.goToConnect();
+
+ // stop adapter and edit adapter
+ ConnectBtns.stopAdapter().click();
+ ConnectBtns.editAdapter().click();
+
+ // change data type of density to integer
+ ConnectBtns.nextBtn().click();
+ ConnectEventSchemaUtils.changePropertyDataType('density', 'Integer');
+
+ ConnectUtils.storeAndStartEditedAdapter();
+
+ // Validate that the data is further persisted in the database by
checking if the amount of events in the data lake changes
+ DataLakeUtils.goToDatalakeConfiguration();
+
+ DataLakeUtils.waitForCountingResults();
+
+ let initialValue;
+
+ DataLakeUtils.getDatalakeNumberOfEvents().then((value) => {
+ initialValue = value;
+ });
+
+ DataLakeBtns.refreshDataLakeMeasures().click();
+
+ DataLakeUtils.waitForCountingResults();
+
+ DataLakeUtils.getDatalakeNumberOfEvents().then((newValue) => {
+ // IMPORTANT: Currently we implemented a workaround by showing the
user a warning message when the data type is changed.
+ // In the future, we need a migration mechanism to automatically
change all the StreamPipes resources that are effected
+ // by the change. Once this is implemented the following line must
be changed to .not.equal.
+ // The issue is tracked here:
https://github.com/apache/streampipes/issues/2954
+ expect(newValue).equal(initialValue);
+ });
+
+ });
});
diff --git a/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts
b/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts
index 8b63cafac1..da5089a227 100644
--- a/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts
+++ b/ui/cypress/tests/adapter/editAdapterSettingsAndPipeline.spec.ts
@@ -70,7 +70,7 @@ describe('Test Edit Adapter and Pipeline', () => {
cy.dataCy('sp-connect-adapter-edit-warning', {
timeout: 60000,
}).should('be.visible');
- cy.dataCy('btn-update-adapter-migrate-pipelines').click();
+ ConnectBtns.updateAndMigratePipelines().click();
ConnectUtils.closeAdapterPreview();
cy.wait(1000);
diff --git a/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
b/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
index 60ce694b21..db78754e74 100644
--- a/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
+++ b/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
@@ -50,7 +50,7 @@ describe('Test Edit Adapter', () => {
.clear()
.type('http://schema.org/Numbers');
// Change field data type
- cy.dataCy('connect-change-runtime-type')
+ ConnectBtns.changeRuntimeType()
.click()
.get('mat-option')
.contains('Double')
@@ -83,7 +83,7 @@ describe('Test Edit Adapter', () => {
'have.value',
'http://schema.org/Numbers',
);
- cy.dataCy('connect-change-runtime-type').should(
+ ConnectBtns.changeRuntimeType().should(
'include.text',
'Double',
);
@@ -96,7 +96,7 @@ describe('Test Edit Adapter', () => {
// Delete inserted values in edit field
cy.dataCy('connect-edit-field-runtime-name').clear();
cy.get('[id="domainproperty"]').clear();
- cy.dataCy('connect-change-runtime-type')
+ ConnectBtns.changeRuntimeType()
.click()
.get('mat-option')
.contains('Float')
diff --git a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
index 6ecb38744b..ad558ab8d2 100644
--- a/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
+++ b/ui/cypress/tests/adapter/rules/deleteTransformationRule.spec.ts
@@ -17,7 +17,7 @@
*/
import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
import { FileManagementUtils } from
'../../../support/utils/FileManagementUtils';
-import { ConnectEventSchemaUtils } from
'../../../support/utils/ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from
'../../../support/utils/connect/ConnectEventSchemaUtils';
import { AdapterBuilder } from '../../../support/builder/AdapterBuilder';
import { ConnectBtns } from '../../../support/utils/connect/ConnectBtns';
diff --git a/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts
b/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts
index 39462cfc06..1aa4d14ce6 100644
--- a/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts
+++ b/ui/cypress/tests/adapter/rules/schemaRules.smoke.spec.ts
@@ -18,7 +18,7 @@
import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
import { FileManagementUtils } from
'../../../support/utils/FileManagementUtils';
-import { ConnectEventSchemaUtils } from
'../../../support/utils/ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from
'../../../support/utils/connect/ConnectEventSchemaUtils';
describe('Connect schema rule transformations', () => {
beforeEach('Setup Test', () => {
diff --git a/ui/cypress/tests/adapter/rules/streamRules.spec.ts
b/ui/cypress/tests/adapter/rules/streamRules.spec.ts
index dbfab9dab7..bf9f946a59 100644
--- a/ui/cypress/tests/adapter/rules/streamRules.spec.ts
+++ b/ui/cypress/tests/adapter/rules/streamRules.spec.ts
@@ -18,7 +18,7 @@
import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
import { FileManagementUtils } from
'../../../support/utils/FileManagementUtils';
-import { ConnectEventSchemaUtils } from
'../../../support/utils/ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from
'../../../support/utils/connect/ConnectEventSchemaUtils';
describe('Connect aggregation rule transformations', () => {
beforeEach('Setup Test', () => {
diff --git a/ui/cypress/tests/adapter/rules/valueRules.ts
b/ui/cypress/tests/adapter/rules/valueRules.ts
index 60ffdbf21a..2163f454d2 100644
--- a/ui/cypress/tests/adapter/rules/valueRules.ts
+++ b/ui/cypress/tests/adapter/rules/valueRules.ts
@@ -18,7 +18,7 @@
import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils';
import { FileManagementUtils } from
'../../../support/utils/FileManagementUtils';
-import { ConnectEventSchemaUtils } from
'../../../support/utils/ConnectEventSchemaUtils';
+import { ConnectEventSchemaUtils } from
'../../../support/utils/connect/ConnectEventSchemaUtils';
describe('Connect value rule transformations', () => {
beforeEach('Setup Test', () => {
diff --git
a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
index b202b2955f..91227a143c 100644
---
a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
+++
b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
@@ -36,7 +36,8 @@
<button
color="accent"
mat-icon-button
- matTooltip="Refresh Schema"
+ matTooltip="Refresh"
+ data-cy='refresh-data-lake-measures'
(click)="loadAvailableMeasurements()"
>
<mat-icon>refresh</mat-icon>
@@ -97,6 +98,7 @@
fxLayoutAlign="center"
style="margin: 10px 0 5px 0"
color="accent"
+ data-cy="datalake-number-of-events-spinner"
*ngIf="configurationEntry.events < 0"
>Loading
</mat-spinner>
