This is an automated email from the ASF dual-hosted git repository.
bossenti pushed a commit to branch
2549-property-renaming-in-the-filestream-adapter-does-not-work
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/2549-property-renaming-in-the-filestream-adapter-does-not-work by
this push:
new 27f70a55e9 fix(#2549): consider renaming rules for timestamp retrieval
27f70a55e9 is described below
commit 27f70a55e9384cefbcdba7c2a2bc2248d93e0f21
Author: bossenti <[email protected]>
AuthorDate: Thu Mar 14 14:12:51 2024 +0100
fix(#2549): consider renaming rules for timestamp retrieval
---
.../iiot/protocol/stream/FileReplayAdapter.java | 232 ++++++++++++++-------
1 file changed, 151 insertions(+), 81 deletions(-)
diff --git
a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
index 744982a888..16f74719a2 100644
---
a/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
+++
b/streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileReplayAdapter.java
@@ -32,6 +32,7 @@ import
org.apache.streampipes.extensions.management.connect.adapter.parser.JsonP
import
org.apache.streampipes.extensions.management.connect.adapter.parser.xml.XmlParser;
import org.apache.streampipes.model.AdapterType;
import org.apache.streampipes.model.connect.guess.GuessSchema;
+import org.apache.streampipes.model.connect.rules.schema.RenameRuleDescription;
import org.apache.streampipes.sdk.StaticProperties;
import org.apache.streampipes.sdk.builder.adapter.AdapterConfigurationBuilder;
import org.apache.streampipes.sdk.helpers.Alternatives;
@@ -48,6 +49,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
+import java.util.List;
+import java.util.Optional;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -74,6 +77,8 @@ public class FileReplayAdapter implements StreamPipesAdapter {
private boolean replaceTimestamp;
private String timestampRuntimeName;
+ private String timestampSourceFieldName;
+
private float speedUp;
private long timestampLastEvent = -1;
@@ -81,118 +86,183 @@ public class FileReplayAdapter implements
StreamPipesAdapter {
@Override
public IAdapterConfiguration declareConfig() {
return AdapterConfigurationBuilder.create(ID, 0, FileReplayAdapter::new)
- .withSupportedParsers(
- new JsonParsers(),
- new CsvParser(),
- new XmlParser(),
- new ImageParser())
- .withAssets(Assets.DOCUMENTATION, Assets.ICON)
- .withLocales(Locales.EN)
- .withCategory(AdapterType.Generic)
- .requiredFile(Labels.withId(FILE_PATH), Filetypes.CSV, Filetypes.JSON,
Filetypes.XML)
- .requiredMultiValueSelection(Labels.withId(REPLACE_TIMESTAMP),
Options.from(""))
- .requiredSingleValueSelection(Labels.withId(REPLAY_ONCE),
Options.from("no", "yes"))
- .requiredAlternatives(Labels.withId(SPEED),
- Alternatives.from(Labels.withId(KEEP_ORIGINAL_TIME), true),
- Alternatives.from(Labels.withId(FASTEST)),
Alternatives.from(Labels.withId(SPEED_UP_FACTOR),
- StaticProperties.group(Labels.withId(SPEED_UP_FACTOR_GROUP),
-
StaticProperties.doubleFreeTextProperty(Labels.withId(SPEED_UP)))))
- .buildConfiguration();
+ .withSupportedParsers(
+ new JsonParsers(),
+ new CsvParser(),
+ new XmlParser(),
+ new ImageParser()
+ )
+ .withAssets(Assets.DOCUMENTATION,
Assets.ICON)
+ .withLocales(Locales.EN)
+ .withCategory(AdapterType.Generic)
+ .requiredFile(
+ Labels.withId(FILE_PATH),
+ Filetypes.CSV,
+ Filetypes.JSON,
+ Filetypes.XML
+ )
+
.requiredMultiValueSelection(Labels.withId(REPLACE_TIMESTAMP), Options.from(""))
+ .requiredSingleValueSelection(
+ Labels.withId(REPLAY_ONCE),
+ Options.from("no", "yes")
+ )
+
.requiredAlternatives(Labels.withId(SPEED),
+
Alternatives.from(Labels.withId(KEEP_ORIGINAL_TIME), true),
+
Alternatives.from(Labels.withId(FASTEST)),
+ Alternatives.from(
+
Labels.withId(SPEED_UP_FACTOR),
+
StaticProperties.group(
+
Labels.withId(SPEED_UP_FACTOR_GROUP),
+
StaticProperties.doubleFreeTextProperty(Labels.withId(
+ SPEED_UP))
+ )
+ )
+ )
+ .buildConfiguration();
}
@Override
- public void onAdapterStarted(IAdapterParameterExtractor extractor,
- IEventCollector collector,
- IAdapterRuntimeContext adapterRuntimeContext)
throws AdapterException {
+ public void onAdapterStarted(
+ IAdapterParameterExtractor extractor,
+ IEventCollector collector,
+ IAdapterRuntimeContext adapterRuntimeContext
+ ) throws AdapterException {
// extract user input
executor = Executors.newScheduledThreadPool(1);
boolean replayOnce = extractor
- .getStaticPropertyExtractor()
- .selectedSingleValue(REPLAY_ONCE, String.class)
- .equals("yes");
+ .getStaticPropertyExtractor()
+ .selectedSingleValue(REPLAY_ONCE, String.class)
+ .equals("yes");
var replaceTimestampStringList = extractor
- .getStaticPropertyExtractor()
- .selectedMultiValues(REPLACE_TIMESTAMP, String.class);
- replaceTimestamp = replaceTimestampStringList.size() != 0;
+ .getStaticPropertyExtractor()
+ .selectedMultiValues(REPLACE_TIMESTAMP, String.class);
+ replaceTimestamp = !replaceTimestampStringList.isEmpty();
var speedUpAlternative = extractor
- .getStaticPropertyExtractor()
- .selectedAlternativeInternalId(SPEED);
+ .getStaticPropertyExtractor()
+ .selectedAlternativeInternalId(SPEED);
speedUp = switch (speedUpAlternative) {
case FASTEST -> Float.MAX_VALUE;
case SPEED_UP_FACTOR -> extractor
- .getStaticPropertyExtractor()
- .singleValueParameter(SPEED_UP, Float.class);
+ .getStaticPropertyExtractor()
+ .singleValueParameter(SPEED_UP, Float.class);
default -> 1.0f;
};
// get timestamp field
var timestampField = extractor
- .getAdapterDescription()
- .getEventSchema()
- .getEventProperties()
- .stream()
- .filter(eventProperty ->
-
eventProperty.getDomainProperties().contains(URI.create(SO.DATE_TIME)))
- .findFirst();
+ .getAdapterDescription()
+ .getEventSchema()
+ .getEventProperties()
+ .stream()
+ .filter(eventProperty ->
+ eventProperty.getDomainProperties()
+ .contains(URI.create(SO.DATE_TIME)))
+ .findFirst();
if (timestampField.isEmpty()) {
throw new AdapterException("Could not find a timestamp field in event
schema");
} else {
- timestampRuntimeName = timestampField.get().getRuntimeName();
+ timestampRuntimeName = timestampField.get()
+ .getRuntimeName();
+ }
+
+ // check for renaming rules that affect the timestamp property
+ // if there is one, we need to use the original field name when extracting
the timestamp from the raw data
+ var renamingRulesTimestamp = getRenamingRulesTimestamp(extractor);
+ if (!renamingRulesTimestamp.isEmpty()) {
+ if (renamingRulesTimestamp.size() == 1) {
+ timestampSourceFieldName = renamingRulesTimestamp.get(0)
+ .getOldRuntimeKey();
+ } else {
+ throw new AdapterException("Invalid configuration - multiple renaming
rules detected which affect the "
+ + "timestamp property.");
+ }
+ } else {
+ // no renaming rules can be found, timestamp name does not change
+ timestampSourceFieldName = timestampRuntimeName;
}
// start replay adapter
if (replayOnce) {
- executor.schedule(() -> parseFile(extractor, collector,
adapterRuntimeContext),
- 0,
- TimeUnit.SECONDS);
+ executor.schedule(
+ () -> parseFile(extractor, collector, adapterRuntimeContext),
+ 0,
+ TimeUnit.SECONDS
+ );
} else {
- executor.scheduleAtFixedRate(() -> parseFile(extractor, collector,
adapterRuntimeContext),
- 0,
- 1,
- TimeUnit.SECONDS);
+ executor.scheduleAtFixedRate(
+ () -> parseFile(extractor, collector, adapterRuntimeContext),
+ 0,
+ 1,
+ TimeUnit.SECONDS
+ );
}
}
- private void parseFile(IAdapterParameterExtractor extractor,
- IEventCollector collector,
- IAdapterRuntimeContext adapterRuntimeContext) {
+ /**
+ * Retrieves a list of {@link RenameRuleDescription} affecting the timestamp
property.
+ *
+ * @param extractor An instance of IAdapterParameterExtractor used to
extract adapter description and rules.
+ * @return A list of {@link RenameRuleDescription} containing rules
affecting timestamp property.
+ */
+ private List<RenameRuleDescription>
getRenamingRulesTimestamp(IAdapterParameterExtractor extractor) {
+ var renamingRules =
+ extractor.getAdapterDescription()
+ .getRules()
+ .stream()
+ .filter(rule -> rule instanceof RenameRuleDescription)
+ .map(rule -> (RenameRuleDescription) rule)
+ .toList();
+
+ return renamingRules.stream()
+ .filter(rule -> rule.getNewRuntimeKey()
+ .equals(timestampRuntimeName))
+ .toList();
+ }
+
+ private void parseFile(
+ IAdapterParameterExtractor extractor,
+ IEventCollector collector,
+ IAdapterRuntimeContext adapterRuntimeContext
+ ) {
try {
InputStream inputStream = getDataFromEndpoint(extractor
- .getStaticPropertyExtractor()
- .selectedFilename(FILE_PATH));
-
- extractor.selectedParser().parse(inputStream, (event) -> {
-
- long actualEventTimestamp = -1;
- if (event.get(timestampRuntimeName) instanceof Long) {
- actualEventTimestamp = (long) event.get(timestampRuntimeName);
- } else {
- LOG.error(
- "The timestamp field is not a unix timestamp in ms. Value: %s"
- .formatted(event.get(timestampRuntimeName)));
- }
-
- if (timestampLastEvent != -1) {
- long sleepTime = (long) ((actualEventTimestamp - timestampLastEvent)
/ speedUp);
- // speed up is set to Float.MAX_VALUE when user selected fastest
option
- if (sleepTime > 0 && speedUp != Float.MAX_VALUE) {
- try {
- Thread.sleep(sleepTime);
- } catch (InterruptedException e) {
- LOG.info("File stream adapter was stopped, the current replay is
interuppted", e);
- }
- }
- }
-
- timestampLastEvent = actualEventTimestamp;
- if (replaceTimestamp) {
- event.put(timestampRuntimeName, System.currentTimeMillis());
- }
-
- collector.collect(event);
- });
+
.getStaticPropertyExtractor()
+
.selectedFilename(FILE_PATH));
+
+ extractor.selectedParser()
+ .parse(inputStream, (event) -> {
+
+ long actualEventTimestamp = -1;
+ if (event.get(timestampSourceFieldName) instanceof Long) {
+ actualEventTimestamp = (long)
event.get(timestampSourceFieldName);
+ } else {
+ LOG.error(
+ "The timestamp field is not a unix timestamp in ms.
Value: %s"
+ .formatted(event.get(timestampSourceFieldName)));
+ }
+
+ if (timestampLastEvent != -1) {
+ long sleepTime = (long) ((actualEventTimestamp -
timestampLastEvent) / speedUp);
+ // speed up is set to Float.MAX_VALUE when user selected
fastest option
+ if (sleepTime > 0 && speedUp != Float.MAX_VALUE) {
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException e) {
+ LOG.info("File stream adapter was stopped, the current
replay is interuppted", e);
+ }
+ }
+ }
+
+ timestampLastEvent = actualEventTimestamp;
+ if (replaceTimestamp) {
+ // we can directly use the desired runtime name for the
timestamp instead of the original here
+ event.put(timestampRuntimeName, System.currentTimeMillis());
+ }
+
+ collector.collect(event);
+ });
} catch (AdapterException e) {
adapterRuntimeContext