This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch
2954-data-type-change-in-adapter-causes-data-storage-failure-in-influxdb
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/2954-data-type-change-in-adapter-causes-data-storage-failure-in-influxdb
by this push:
new ccf2fb1574 fix(#2954): Add warning message to data type change if
adapter is edited
ccf2fb1574 is described below
commit ccf2fb1574bec2db27710993f921d2f57cec1598
Author: Philipp Zehnder <[email protected]>
AuthorDate: Wed Jun 26 10:19:21 2024 +0200
fix(#2954): Add warning message to data type change if adapter is edited
---
ui/cypress/support/utils/connect/ConnectBtns.ts | 1 -
.../support/utils/connect/ConnectEventSchemaUtils.ts | 16 +++++++++++++---
ui/cypress/support/utils/datalake/DataLakeBtns.ts | 3 +--
ui/cypress/support/utils/datalake/DataLakeUtils.ts | 16 +++++++++-------
ui/cypress/tests/adapter/editAdapter.smoke.spec.ts | 11 +++++++----
.../tests/adapter/editAdapterValuesAndFields.spec.ts | 5 +----
.../datalake-configuration.component.html | 2 +-
.../edit-schema-transformation.component.html | 15 +++++++++++++++
.../edit-schema-transformation.component.scss | 2 ++
.../edit-schema-transformation.component.ts | 6 ++++++
.../edit-correction-value.component.scss | 8 +-------
.../edit-event-property.component.scss | 8 ++++++++
12 files changed, 64 insertions(+), 29 deletions(-)
diff --git a/ui/cypress/support/utils/connect/ConnectBtns.ts
b/ui/cypress/support/utils/connect/ConnectBtns.ts
index 5b18603a2e..f828bcf148 100644
--- a/ui/cypress/support/utils/connect/ConnectBtns.ts
+++ b/ui/cypress/support/utils/connect/ConnectBtns.ts
@@ -52,7 +52,6 @@ export class ConnectBtns {
return cy.get('button').contains('Next').parent();
}
-
// ===================== Adapter settings btns ==========================
public static adapterSettingsStartAdapter() {
return cy.dataCy('adapter-settings-start-adapter-btn');
diff --git a/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
b/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
index 3d84b50380..098ea68554 100644
--- a/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
+++ b/ui/cypress/support/utils/connect/ConnectEventSchemaUtils.ts
@@ -198,8 +198,12 @@ export class ConnectEventSchemaUtils {
public static changePropertyDataType(
propertyName: string,
dataType: string,
+ warningIsShown: boolean = false,
) {
ConnectEventSchemaUtils.clickEditProperty(propertyName);
+
+ this.checkIfWarningIsShown(warningIsShown);
+
ConnectBtns.changeRuntimeType()
.click()
.get('mat-option')
@@ -210,13 +214,19 @@ export class ConnectEventSchemaUtils {
cy.dataCy('edit-' + propertyName, { timeout: 10000 }).click({
force: true,
});
- ConnectBtns.changeRuntimeType().contains(
- dataType,
- );
+ ConnectBtns.changeRuntimeType().contains(dataType);
cy.wait(1000);
cy.dataCy('sp-save-edit-property').click();
}
+ private static checkIfWarningIsShown(warningIsShown: boolean) {
+ if (warningIsShown) {
+ cy.dataCy('warning-change-data-type').should('be.visible');
+ } else {
+ cy.dataCy('warning-change-data-type').should('not.exist');
+ }
+ }
+
public static eventSchemaNextBtnDisabled() {
cy.get('#event-schema-next-button').should('be.disabled');
}
diff --git a/ui/cypress/support/utils/datalake/DataLakeBtns.ts
b/ui/cypress/support/utils/datalake/DataLakeBtns.ts
index 03ab7e1759..5bb7083108 100644
--- a/ui/cypress/support/utils/datalake/DataLakeBtns.ts
+++ b/ui/cypress/support/utils/datalake/DataLakeBtns.ts
@@ -17,8 +17,7 @@
*/
export class DataLakeBtns {
-
public static refreshDataLakeMeasures() {
return cy.dataCy('refresh-data-lake-measures');
}
- }
+}
diff --git a/ui/cypress/support/utils/datalake/DataLakeUtils.ts
b/ui/cypress/support/utils/datalake/DataLakeUtils.ts
index 064a73b494..41fadcf9b0 100644
--- a/ui/cypress/support/utils/datalake/DataLakeUtils.ts
+++ b/ui/cypress/support/utils/datalake/DataLakeUtils.ts
@@ -333,18 +333,20 @@ export class DataLakeUtils {
return currentDate;
}
-
public static waitForCountingResults() {
- cy.dataCy('datalake-number-of-events-spinner', { timeout: 10000 })
- .should('exist');
- cy.dataCy('datalake-number-of-events-spinner', { timeout: 10000 })
- .should('not.exist');
+ cy.dataCy('datalake-number-of-events-spinner', {
+ timeout: 10000,
+ }).should('exist');
+ cy.dataCy('datalake-number-of-events-spinner', {
+ timeout: 10000,
+ }).should('not.exist');
}
public static getDatalakeNumberOfEvents(): Cypress.Chainable<string> {
- return cy.dataCy('datalake-number-of-events', { timeout: 10000 })
+ return cy
+ .dataCy('datalake-number-of-events', { timeout: 10000 })
.should('be.visible')
.invoke('text')
- .then((text) => text.trim());
+ .then(text => text.trim());
}
}
diff --git a/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
b/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
index b346273e24..3becf859e8 100644
--- a/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
+++ b/ui/cypress/tests/adapter/editAdapter.smoke.spec.ts
@@ -83,7 +83,11 @@ describe('Test Edit Adapter', () => {
// change data type of density to integer
ConnectBtns.nextBtn().click();
- ConnectEventSchemaUtils.changePropertyDataType('density', 'Integer');
+ ConnectEventSchemaUtils.changePropertyDataType(
+ 'density',
+ 'Integer',
+ true,
+ );
ConnectUtils.storeAndStartEditedAdapter();
@@ -94,7 +98,7 @@ describe('Test Edit Adapter', () => {
let initialValue;
- DataLakeUtils.getDatalakeNumberOfEvents().then((value) => {
+ DataLakeUtils.getDatalakeNumberOfEvents().then(value => {
initialValue = value;
});
@@ -102,13 +106,12 @@ describe('Test Edit Adapter', () => {
DataLakeUtils.waitForCountingResults();
- DataLakeUtils.getDatalakeNumberOfEvents().then((newValue) => {
+ DataLakeUtils.getDatalakeNumberOfEvents().then(newValue => {
// IMPORTANT: Currently we implemented a workaround by showing the
user a warning message when the data type is changed.
// In the future, we need a migration mechanism to automatically
change all the StreamPipes resources that are effected
// by the change. Once this is implemented the following line must
be changed to .not.equal.
// The issue is tracked here:
https://github.com/apache/streampipes/issues/2954
expect(newValue).equal(initialValue);
});
-
});
});
diff --git a/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
b/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
index db78754e74..52269352a0 100644
--- a/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
+++ b/ui/cypress/tests/adapter/editAdapterValuesAndFields.spec.ts
@@ -83,10 +83,7 @@ describe('Test Edit Adapter', () => {
'have.value',
'http://schema.org/Numbers',
);
- ConnectBtns.changeRuntimeType().should(
- 'include.text',
- 'Double',
- );
+ ConnectBtns.changeRuntimeType().should('include.text', 'Double');
cy.dataCy('connect-schema-correction-value').should('have.value', '2');
cy.dataCy('connect-schema-correction-operator').should(
'include.text',
diff --git
a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
index 91227a143c..5044fd3091 100644
---
a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
+++
b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
@@ -37,7 +37,7 @@
color="accent"
mat-icon-button
matTooltip="Refresh"
- data-cy='refresh-data-lake-measures'
+ data-cy="refresh-data-lake-measures"
(click)="loadAvailableMeasurements()"
>
<mat-icon>refresh</mat-icon>
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.html
b/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.html
index 93b6b012c2..84732adcc3 100644
---
a/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.html
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.html
@@ -92,6 +92,21 @@
sectionDescription="The data type of the field values"
*ngIf="!isTimestampProperty && !isNestedProperty"
>
+ <div *ngIf="adapterIsInEditMode" class="warning-message">
+ <p data-cy="warning-change-data-type" class="text-center">
+ Changing the data type can result in pipelines no longer
writing
+ the data correctly to the data lake.
+ <br />
+ <br />
+ To resolve this problem, you can:
+ <br />
+ - Rename the property.
+ <br />
+ - Truncate the old data.
+ <br />
+ - Write the data to a new measurement.
+ </p>
+ </div>
<sp-edit-data-type
[cachedProperty]="cachedProperty"
(dataTypeChanged)="dataTypeChanged.emit($event)"
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.scss
b/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.scss
index 13cbc4aacb..0d4d02f4cc 100644
---
a/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.scss
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.scss
@@ -15,3 +15,5 @@
* limitations under the License.
*
*/
+
+@import '../../edit-event-property.component.scss';
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.ts
b/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.ts
index 80adb96525..608b6c4124 100644
---
a/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.ts
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-schema-transformation/edit-schema-transformation.component.ts
@@ -31,6 +31,7 @@ import {
SemanticType,
SemanticTypesRestService,
} from '@streampipes/platform-services';
+import { Router } from '@angular/router';
@Component({
selector: 'sp-edit-schema-transformation',
@@ -52,9 +53,12 @@ export class EditSchemaTransformationComponent implements
OnInit {
domainPropertyControl = new UntypedFormControl();
semanticTypes: Observable<string[]>;
+ adapterIsInEditMode: boolean;
+
constructor(
private semanticTypesRestService: SemanticTypesRestService,
private shepherdService: ShepherdService,
+ private router: Router,
) {}
ngOnInit(): void {
@@ -68,6 +72,8 @@ export class EditSchemaTransformationComponent implements
OnInit {
: [];
}),
);
+
+ this.adapterIsInEditMode = this.router.url.includes('connect/edit');
}
editTimestampDomainProperty(checked: boolean) {
diff --git
a/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-correction-value/edit-correction-value.component.scss
b/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-correction-value/edit-correction-value.component.scss
index ec202a065d..a1d59b6014 100644
---
a/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-correction-value/edit-correction-value.component.scss
+++
b/ui/src/app/connect/dialog/edit-event-property/components/edit-value-transformation/edit-correction-value/edit-correction-value.component.scss
@@ -16,10 +16,4 @@
*
*/
-.warning-message {
- margin-bottom: 10px;
- padding: 5px;
- border: 1px solid var(--color-warn);
- background: var(--color-bg-1);
- font-size: 14px;
-}
+@import '../../../edit-event-property.component.scss';
diff --git
a/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.scss
b/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.scss
index b92834fe68..0c215ed64a 100644
---
a/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.scss
+++
b/ui/src/app/connect/dialog/edit-event-property/edit-event-property.component.scss
@@ -29,3 +29,11 @@
.full-width {
width: 100%;
}
+
+.warning-message {
+ margin-bottom: 10px;
+ padding: 5px;
+ border: 1px solid var(--color-warn);
+ background: var(--color-bg-1);
+ font-size: 14px;
+}
