This is an automated email from the ASF dual-hosted git repository. zehnder pushed a commit to branch STREAMPIPES-612 in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit f45c9d2304889a9bbcc44978f0f31c002a495bec Author: Philipp Zehnder <[email protected]> AuthorDate: Tue Nov 1 21:55:44 2022 +0100 [STREAMPIPES-612] Add builder pattern for GenericAdapterStreamDescriptions --- .../format/json/object/JsonObjectFormat.java | 9 +- .../integration/adapters/PulsarAdapterTester.java | 156 ++++++++++----------- .../adapter/GenericAdapterStreamDescription.java | 73 +++++----- .../builder/adapter/AdapterDescriptionBuilder.java | 2 +- .../adapter/GenericDataStreamAdapterBuilder.java | 67 ++++++++- .../GenericDataStreamAdapterBuilderTest.java | 88 ++++++++++++ 6 files changed, 273 insertions(+), 122 deletions(-) diff --git a/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/format/json/object/JsonObjectFormat.java b/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/format/json/object/JsonObjectFormat.java index f5306bbaf..0a86d07d4 100644 --- a/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/format/json/object/JsonObjectFormat.java +++ b/streampipes-connect/src/main/java/org/apache/streampipes/connect/adapter/format/json/object/JsonObjectFormat.java @@ -18,14 +18,15 @@ package org.apache.streampipes.connect.adapter.format.json.object; -import org.apache.streampipes.connect.api.IFormat; import org.apache.streampipes.connect.adapter.format.json.AbstractJsonFormat; +import org.apache.streampipes.connect.api.IFormat; import org.apache.streampipes.model.connect.grounding.FormatDescription; import org.apache.streampipes.sdk.builder.adapter.FormatDescriptionBuilder; public class JsonObjectFormat extends AbstractJsonFormat { public static final String ID = "https://streampipes.org/vocabulary/v1/format/json/object"; + public static final String LABEL = "Single Object"; @Override public IFormat getInstance(FormatDescription formatDescription) { @@ -34,9 +35,9 @@ public class JsonObjectFormat extends AbstractJsonFormat { @Override public FormatDescription declareModel() { - return FormatDescriptionBuilder.create(ID, "Single Object", "Each event is a single json object (e.g. {'value': 1})") - .addFormatType(JSON_FORMAT_TYPE) - .build(); + return FormatDescriptionBuilder.create(ID, LABEL, "Each event is a single json object (e.g. {'value': 1})") + .addFormatType(JSON_FORMAT_TYPE) + .build(); } @Override diff --git a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/adapters/PulsarAdapterTester.java b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/adapters/PulsarAdapterTester.java index 4ac2c90da..566a835e2 100644 --- a/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/adapters/PulsarAdapterTester.java +++ b/streampipes-integration-tests/src/test/java/org/apache/streampipes/integration/adapters/PulsarAdapterTester.java @@ -18,110 +18,102 @@ package org.apache.streampipes.integration.adapters; import com.google.common.collect.Maps; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClient; -import org.apache.streampipes.connect.adapter.format.json.arraykey.JsonFormat; +import org.apache.streampipes.connect.adapter.format.json.object.JsonObjectFormat; import org.apache.streampipes.connect.api.exception.AdapterException; import org.apache.streampipes.connect.iiot.protocol.stream.pulsar.PulsarProtocol; import org.apache.streampipes.integration.containers.PulsarContainer; import org.apache.streampipes.integration.containers.PulsarDevContainer; import org.apache.streampipes.model.connect.adapter.AdapterDescription; -import org.apache.streampipes.model.connect.adapter.GenericAdapterStreamDescription; -import org.apache.streampipes.model.connect.grounding.FormatDescription; -import org.apache.streampipes.model.connect.grounding.ProtocolDescription; import org.apache.streampipes.model.connect.rules.DebugSinkRuleDescription; -import org.apache.streampipes.model.staticproperty.FreeTextStaticProperty; +import org.apache.streampipes.sdk.builder.adapter.GenericDataStreamAdapterBuilder; +import org.apache.streampipes.sdk.builder.adapter.ProtocolDescriptionBuilder; +import org.apache.streampipes.sdk.helpers.Labels; import org.junit.Assert; import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + public class PulsarAdapterTester extends AdapterTesterBase { - PulsarContainer pulsarContainer; - private final static String TOPIC = "test-topic"; + PulsarContainer pulsarContainer; + private final static String TOPIC = "test-topic"; - @Override - public void startAdapterService() { - if (Objects.equals(System.getenv("TEST_MODE"), "dev")) { - pulsarContainer = new PulsarDevContainer(); - } else { - pulsarContainer = new PulsarContainer(); - } - pulsarContainer.start(); + @Override + public void startAdapterService() { + if (Objects.equals(System.getenv("TEST_MODE"), "dev")) { + pulsarContainer = new PulsarDevContainer(); + } else { + pulsarContainer = new PulsarContainer(); } + pulsarContainer.start(); + } - @Override - public AdapterDescription prepareAdapter() { - GenericAdapterStreamDescription description = new GenericAdapterStreamDescription(); + @Override + public AdapterDescription prepareAdapter() { - // Configure format - FormatDescription formatDescription = new FormatDescription(); - formatDescription.setAppId(JsonFormat.ID); - var f = new FreeTextStaticProperty("key", "", ""); - f.setValue("timestamp"); - formatDescription.addConfig(f); - description.setFormatDescription(formatDescription); + return GenericDataStreamAdapterBuilder + .create(PulsarProtocol.ID) + .format(new JsonObjectFormat() + .declareModel()) + .protocol(ProtocolDescriptionBuilder.create(PulsarProtocol.ID) + .requiredTextParameter( + Labels.withId(PulsarProtocol.PULSAR_BROKER_HOST), + pulsarContainer.getBrokerHost()) + .requiredIntegerParameter( + Labels.withId(PulsarProtocol.PULSAR_BROKER_PORT), + pulsarContainer.getBrokerPort()) + .requiredTextParameter( + Labels.withId(PulsarProtocol.PULSAR_TOPIC), TOPIC) + .build()) + .addRules(List.of(new DebugSinkRuleDescription())) + .build(); + } - // Configure protocol - ProtocolDescription protocolDescription = new ProtocolDescription(); - protocolDescription.addConfig(FreeTextStaticProperty.of(PulsarProtocol.PULSAR_BROKER_HOST, - pulsarContainer.getBrokerHost())); - protocolDescription.addConfig(FreeTextStaticProperty.of(PulsarProtocol.PULSAR_BROKER_PORT, - pulsarContainer.getBrokerPort().toString())); - protocolDescription.addConfig(FreeTextStaticProperty.of(PulsarProtocol.PULSAR_TOPIC, TOPIC)); - protocolDescription.setAppId(PulsarProtocol.ID); - description.setProtocolDescription(protocolDescription); + @Override + public List<Map<String, Object>> generateData() throws Exception { + List<Map<String, Object>> result = new ArrayList<>(); + try (PulsarClient client = PulsarClient.builder().serviceUrl( + String.format("pulsar://%s:%s", pulsarContainer.getBrokerHost(), + pulsarContainer.getBrokerPort())) + .build(); + Producer<byte[]> producer = client.newProducer().topic(TOPIC).create()) { + ObjectMapper objectMapper = new ObjectMapper(); - // Configure debug sink rule - description.setRules(Collections.singletonList(new DebugSinkRuleDescription())); - - return description; + for (int i = 0; i < 3; i++) { + var dataMap = new HashMap<String, Object>(); + dataMap.put("timestamp", i); + dataMap.put("value", "test-data"); + byte[] data = objectMapper.writeValueAsBytes(dataMap); + producer.sendAsync(data); + result.add(dataMap); + } + producer.flush(); } + return result; + } - @Override - public List<Map<String, Object>> generateData() throws Exception { - List<Map<String, Object>> result = new ArrayList<>(); - try (PulsarClient client = PulsarClient.builder().serviceUrl( - String.format("pulsar://%s:%s", pulsarContainer.getBrokerHost(), - pulsarContainer.getBrokerPort())) - .build(); - Producer<byte[]> producer = client.newProducer().topic(TOPIC).create()) { - ObjectMapper objectMapper = new ObjectMapper(); - - for (int i = 0; i < 3; i++) { - var dataMap = new HashMap<String, Object>(); - dataMap.put("timestamp", i); - dataMap.put("value", "test-data"); - byte[] data = objectMapper.writeValueAsBytes(dataMap); - producer.sendAsync(data); - result.add(dataMap); - } - producer.flush(); - } - return result; + @Override + public void validateData(List<Map<String, Object>> expectedData) throws Exception { + for (Map<String, Object> expected : expectedData) { + Map<String, Object> actual = takeEvent(); + Assert.assertTrue(Maps.difference(expected, actual).areEqual()); } + } - @Override - public void validateData(List<Map<String, Object>> expectedData) throws Exception { - for (Map<String, Object> expected : expectedData) { - Map<String, Object> actual = takeEvent(); - Assert.assertTrue(Maps.difference(expected, actual).areEqual()); - } + @Override + public void close() { + if (pulsarContainer != null) { + pulsarContainer.stop(); } - - @Override - public void close() { - if (pulsarContainer != null) { - pulsarContainer.stop(); - } - try { - stopAdapter(); - } catch (AdapterException e) { - throw new RuntimeException(e); - } + try { + stopAdapter(); + } catch (AdapterException e) { + throw new RuntimeException(e); } + } } diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/GenericAdapterStreamDescription.java b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/GenericAdapterStreamDescription.java index 98aea2c8e..b116caae0 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/GenericAdapterStreamDescription.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/connect/adapter/GenericAdapterStreamDescription.java @@ -26,52 +26,57 @@ import org.apache.streampipes.model.shared.annotation.TsModel; import org.apache.streampipes.model.util.ElementIdGenerator; @TsModel -public class GenericAdapterStreamDescription extends AdapterStreamDescription implements GenericAdapterDescription{ +public class GenericAdapterStreamDescription extends AdapterStreamDescription implements GenericAdapterDescription { - public static final String ID = ElementIdGenerator.makeFixedElementId(GenericAdapterStreamDescription.class); + public static final String ID = ElementIdGenerator.makeFixedElementId(GenericAdapterStreamDescription.class); - private FormatDescription formatDescription; + private FormatDescription formatDescription; - private ProtocolDescription protocolDescription; + private ProtocolDescription protocolDescription; - public GenericAdapterStreamDescription() { - super(ID, "GenericAdapterStreamDescription", ""); - } + public GenericAdapterStreamDescription() { + super(ID, "GenericAdapterStreamDescription", ""); + } - public GenericAdapterStreamDescription(FormatDescription formatDescription, ProtocolStreamDescription protocolDescription) { - this.formatDescription = formatDescription; - this.protocolDescription = protocolDescription; - } + public GenericAdapterStreamDescription(FormatDescription formatDescription, + ProtocolStreamDescription protocolDescription) { + this.formatDescription = formatDescription; + this.protocolDescription = protocolDescription; + } - public GenericAdapterStreamDescription(GenericAdapterStreamDescription other) { - super(other); + public GenericAdapterStreamDescription(GenericAdapterStreamDescription other) { + super(other); - if (other.getFormatDescription() != null) this.formatDescription = new FormatDescription(other.getFormatDescription()); - if (other.getProtocolDescription() != null) this.protocolDescription = new ProtocolStreamDescription(other.getProtocolDescription()); - } + if (other.getFormatDescription() != null) { + this.formatDescription = new FormatDescription(other.getFormatDescription()); + } + if (other.getProtocolDescription() != null) { + this.protocolDescription = new ProtocolStreamDescription(other.getProtocolDescription()); + } + } - public FormatDescription getFormatDescription() { - return formatDescription; - } + public FormatDescription getFormatDescription() { + return formatDescription; + } - @Override - public EventSchema getEventSchema() { - if (this.getDataStream() != null) { - return this.getDataStream().getEventSchema(); - } - return null; + @Override + public EventSchema getEventSchema() { + if (this.getDataStream() != null) { + return this.getDataStream().getEventSchema(); } + return null; + } - public void setFormatDescription(FormatDescription formatDescription) { - this.formatDescription = formatDescription; - } + public void setFormatDescription(FormatDescription formatDescription) { + this.formatDescription = formatDescription; + } - public ProtocolDescription getProtocolDescription() { - return protocolDescription; - } + public ProtocolDescription getProtocolDescription() { + return protocolDescription; + } - public void setProtocolDescription(ProtocolDescription protocolDescription) { - this.protocolDescription = protocolDescription; - } + public void setProtocolDescription(ProtocolDescription protocolDescription) { + this.protocolDescription = protocolDescription; + } } diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterDescriptionBuilder.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterDescriptionBuilder.java index db18910a4..0843a755a 100644 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterDescriptionBuilder.java +++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/AdapterDescriptionBuilder.java @@ -37,7 +37,7 @@ public abstract class AdapterDescriptionBuilder<BU extends super(id, label, description, adapterTypeInstance); } - public AdapterDescriptionBuilder<BU, T> category(AdapterType... categories) { + public BU category(AdapterType... categories) { this.elementDescription .setCategory(Arrays .stream(categories) diff --git a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/GenericDataStreamAdapterBuilder.java b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/GenericDataStreamAdapterBuilder.java index b9c367775..9d5f2f07f 100644 --- a/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/GenericDataStreamAdapterBuilder.java +++ b/streampipes-sdk/src/main/java/org/apache/streampipes/sdk/builder/adapter/GenericDataStreamAdapterBuilder.java @@ -17,5 +17,70 @@ */ package org.apache.streampipes.sdk.builder.adapter; -public class GenericDataStreamAdapterBuilder { +import org.apache.streampipes.model.connect.adapter.GenericAdapterStreamDescription; +import org.apache.streampipes.model.connect.grounding.FormatDescription; +import org.apache.streampipes.model.connect.grounding.ProtocolDescription; +import org.apache.streampipes.model.connect.rules.TransformationRuleDescription; + +import java.util.List; + +public class GenericDataStreamAdapterBuilder extends + AdapterDescriptionBuilder<GenericDataStreamAdapterBuilder, GenericAdapterStreamDescription> { + + protected GenericDataStreamAdapterBuilder(String id, GenericAdapterStreamDescription element) { + super(id, element); + } + + /** + * Create a new generic stream adapter description using the builder pattern. + * + * @param appId that is set for the GenericAdapterStreamDescription + * @return the builder object + */ + public static GenericDataStreamAdapterBuilder create(String appId) { + return new GenericDataStreamAdapterBuilder(appId, new GenericAdapterStreamDescription()); + } + + @Override + protected GenericDataStreamAdapterBuilder me() { + return this; + } + + @Override + protected void prepareBuild() { + + } + + /** + * Add a FormatDescription to the GenericDataStreamAdapterDescription + * + * @param formatDescription that should be added to the adapter + * @return the builder object + */ + public GenericDataStreamAdapterBuilder format(FormatDescription formatDescription) { + this.elementDescription.setFormatDescription(formatDescription); + return this; + } + + /** + * Add a ProtocolDescription to the GenericDataStreamAdapterDescription + * + * @param protocolDescription that should be added to the adapter + * @return the builder object + */ + public GenericDataStreamAdapterBuilder protocol(ProtocolDescription protocolDescription) { + this.elementDescription.setProtocolDescription(protocolDescription); + return this; + } + + /** + * Add a list of TransformationRules to the GenericDataStreamAdapterDescription + * + * @param rules that should be added to the adapter + * @return the builder object + */ + public GenericDataStreamAdapterBuilder addRules(List<TransformationRuleDescription> rules) { + this.elementDescription.setRules(rules); + return this; + } } diff --git a/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/builder/adapter/GenericDataStreamAdapterBuilderTest.java b/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/builder/adapter/GenericDataStreamAdapterBuilderTest.java new file mode 100644 index 000000000..5168ecefe --- /dev/null +++ b/streampipes-sdk/src/test/java/org/apache/streampipes/sdk/builder/adapter/GenericDataStreamAdapterBuilderTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.sdk.builder.adapter; + +import org.apache.streampipes.model.AdapterType; +import org.apache.streampipes.model.connect.grounding.FormatDescription; +import org.apache.streampipes.model.connect.grounding.ProtocolDescription; +import org.apache.streampipes.model.connect.rules.value.AddTimestampRuleDescription; +import org.apache.streampipes.sdk.helpers.Locales; +import org.apache.streampipes.sdk.utils.Assets; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class GenericDataStreamAdapterBuilderTest { + + private static final String ID = "org.apache.streampipes.sdk.builder.adapter.id"; + + GenericDataStreamAdapterBuilder basicBuilder = + GenericDataStreamAdapterBuilder.create(ID).withAssets(Assets.DOCUMENTATION, Assets.ICON).withLocales(Locales.EN) + .category(AdapterType.Manufacturing); + + @Test + public void basicDescriptionParameterTests() { + var result = basicBuilder.build(); + + assertEquals(ID, result.getAppId()); + assertTrue(result.getIncludedAssets().contains(Assets.DOCUMENTATION)); + assertTrue(result.getIncludedAssets().contains(Assets.ICON)); + assertTrue(result.getIncludedLocales().contains(Locales.EN.toFilename())); + assertTrue(result.getCategory().contains(AdapterType.Manufacturing.name())); + } + + @Test + public void formatDescriptionTest() { + FormatDescription formatDescription = FormatDescriptionBuilder.create("", "", "").build(); + + var result = basicBuilder + .format(formatDescription) + .build(); + + assertEquals(formatDescription, result.getFormatDescription()); + } + + @Test + public void protocolDescriptionTest() { + ProtocolDescription protocolDescription = ProtocolDescriptionBuilder + .create("") + .build(); + + var result = basicBuilder + .protocol(protocolDescription) + .build(); + assertEquals(protocolDescription, result.getProtocolDescription()); + } + + @Test + public void addRuleTest() { + var testRule = new AddTimestampRuleDescription(); + + var result = basicBuilder + .addRules(List.of(testRule)) + .build(); + + assertEquals(1, result.getRules().size()); + assertEquals(testRule, result.getRules().get(0)); + } + +} \ No newline at end of file
