This is an automated email from the ASF dual-hosted git repository. tmitsch pushed a commit to branch feature/improve-scraper-tim in repository https://gitbox.apache.org/repos/asf/plc4x.git
commit 1c1802b2a06ccbd2db96171cc2bb98208616125c Author: Tim Mitsch <[email protected]> AuthorDate: Sun Aug 11 19:37:16 2019 +0200 some refactoring --- .../java/org/apache/plc4x/Plc4XBaseTableTest.java | 4 +- .../org/apache/plc4x/java/scraper/ScrapeJob.java | 7 +- .../apache/plc4x/java/scraper/ScrapeJobImpl.java | 7 +- .../apache/plc4x/java/scraper/ScraperTaskImpl.java | 47 ++++++--- .../java/scraper/config/JobConfiguration.java | 7 +- .../config/JobConfigurationClassicImpl.java | 49 +++++++++ ...ava => JobConfigurationClassicImplBuilder.java} | 20 ++-- .../java/scraper/config/JobConfigurationImpl.java | 35 ++++--- .../java/scraper/config/ScraperConfiguration.java | 97 ++---------------- .../config/ScraperConfigurationClassicImpl.java | 86 ++++++++++++++++ ...=> ScraperConfigurationClassicImplBuilder.java} | 16 +-- .../JobConfigurationTriggeredImpl.java | 47 +++++++++ ...a => JobConfigurationTriggeredImplBuilder.java} | 16 +-- ...java => ScraperConfigurationTriggeredImpl.java} | 99 +++++++----------- ... ScraperConfigurationTriggeredImplBuilder.java} | 18 ++-- .../TriggeredJobConfiguration.java | 74 -------------- .../triggeredscraper/TriggeredScrapeJobImpl.java | 3 +- .../triggeredscraper/TriggeredScraperImpl.java | 53 +++++++--- .../triggeredscraper/TriggeredScraperTask.java | 39 +++---- .../triggerhandler/TriggerConfiguration.java | 99 +++++++++--------- .../triggerhandler/collector/TriggerCollector.java | 3 +- .../collector/TriggerCollectorImpl.java | 112 +++++++++------------ .../java/scraper/ScraperConfigurationTest.java | 21 ++-- .../apache/plc4x/java/scraper/ScraperRunner.java | 3 +- .../plc4x/java/scraper/TriggeredScraperRunner.java | 5 +- .../java/scraper/TriggeredScraperRunnerModbus.java | 5 +- .../config/ScraperConfigurationBuilderTest.java | 12 +-- .../triggeredscraper/TriggeredScraperImplTest.java | 7 +- .../triggerhandler/TriggerConfigurationTest.java | 5 +- 29 files changed, 527 insertions(+), 469 deletions(-) diff --git a/plc4j/integrations/apache-calcite/src/test/java/org/apache/plc4x/Plc4XBaseTableTest.java b/plc4j/integrations/apache-calcite/src/test/java/org/apache/plc4x/Plc4XBaseTableTest.java index 3bc3007..d80cc6b 100644 --- a/plc4j/integrations/apache-calcite/src/test/java/org/apache/plc4x/Plc4XBaseTableTest.java +++ b/plc4j/integrations/apache-calcite/src/test/java/org/apache/plc4x/Plc4XBaseTableTest.java @@ -19,7 +19,7 @@ under the License. package org.apache.plc4x; import org.apache.calcite.linq4j.Enumerator; -import org.apache.plc4x.java.scraper.config.JobConfigurationImpl; +import org.apache.plc4x.java.scraper.config.JobConfigurationClassicImpl; import org.assertj.core.api.WithAssertions; import org.junit.jupiter.api.Test; @@ -33,7 +33,7 @@ class Plc4XBaseTableTest implements WithAssertions { @Test void testOnBlockingQueue() { ArrayBlockingQueue<Plc4xSchema.Record> queue = new ArrayBlockingQueue<>(100); - Plc4xStreamTable table = new Plc4xStreamTable(queue, new JobConfigurationImpl("job1", 100, + Plc4xStreamTable table = new Plc4xStreamTable(queue, new JobConfigurationClassicImpl("job1", 100, Collections.emptyList(), Collections.singletonMap("key", "address"))); diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java index dd46f5f..6cde8d6 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJob.java @@ -19,12 +19,15 @@ package org.apache.plc4x.java.scraper; +import org.apache.plc4x.java.scraper.config.JobConfigurationClassicImpl; +import org.apache.plc4x.java.scraper.config.ScraperConfigurationClassicImpl; + import java.util.Map; /** * POJO Object to transport all Job information. - * Is generated from {@link org.apache.plc4x.java.scraper.config.ScraperConfiguration} by - * merging the sources and the {@link org.apache.plc4x.java.scraper.config.JobConfigurationImpl}. + * Is generated from {@link ScraperConfigurationClassicImpl} by + * merging the sources and the {@link JobConfigurationClassicImpl}. */ public interface ScrapeJob { String getJobName(); diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJobImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJobImpl.java index b541e47..9b0c181 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJobImpl.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScrapeJobImpl.java @@ -19,14 +19,15 @@ package org.apache.plc4x.java.scraper; -import org.apache.plc4x.java.scraper.config.JobConfigurationImpl; +import org.apache.plc4x.java.scraper.config.JobConfigurationClassicImpl; +import org.apache.plc4x.java.scraper.config.ScraperConfigurationClassicImpl; import java.util.Map; /** * POJO Object to transport all Job information. - * Is generated from {@link org.apache.plc4x.java.scraper.config.ScraperConfiguration} by - * merging the sources and the {@link JobConfigurationImpl}. + * Is generated from {@link ScraperConfigurationClassicImpl} by + * merging the sources and the {@link JobConfigurationClassicImpl}. * * @deprecated Scraper is deprecated please use {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScrapeJobImpl} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} */ diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTaskImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTaskImpl.java index f15ee84..4c5a270 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTaskImpl.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/ScraperTaskImpl.java @@ -29,7 +29,7 @@ import org.apache.plc4x.java.api.exceptions.PlcRuntimeException; import org.apache.plc4x.java.api.messages.PlcReadRequest; import org.apache.plc4x.java.api.messages.PlcReadResponse; import org.apache.plc4x.java.api.types.PlcResponseCode; -import org.apache.plc4x.java.scraper.config.JobConfigurationImpl; +import org.apache.plc4x.java.scraper.config.JobConfigurationClassicImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +45,7 @@ import java.util.stream.Collectors; /** * Plc Scraper Task that scrapes one source. * One {@link ScrapeJobImpl} gets split into multiple tasks. - * One task for each source that is defined in the {@link JobConfigurationImpl}. + * One task for each source that is defined in the {@link JobConfigurationClassicImpl}. * * @deprecated Scraper is deprecated please use {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScrapeJobImpl} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} */ @@ -93,10 +93,13 @@ public class ScraperTaskImpl implements ScraperTask { this.resultHandler = resultHandler; } + @Override public void run() { // Does a single fetch - LOGGER.trace("Start new scrape of task of job {} for connection {}", jobName, connectionAlias); + if(LOGGER.isDebugEnabled()) { + LOGGER.debug("Start new scrape of task of job {} for connection {}", jobName, connectionAlias); + } requestCounter.incrementAndGet(); StopWatch stopWatch = new StopWatch(); stopWatch.start(); @@ -111,15 +114,17 @@ public class ScraperTaskImpl implements ScraperTask { } }, handlerService); connection = future.get(10*requestTimeoutMs, TimeUnit.MILLISECONDS); - LOGGER.trace("Connection to {} established: {}", connectionString, connection); - PlcReadResponse response; + LOGGER.debug("Connection to {} established: {}", connectionString, connection); + PlcReadResponse plcReadResponse; try { - PlcReadRequest.Builder builder = connection.readRequestBuilder(); - fields.forEach((alias,qry) -> { + //build read request + PlcReadRequest.Builder readRequestBuilder = connection.readRequestBuilder(); + //add fields to be acquired to builder + fields.forEach((alias, qry) -> { LOGGER.trace("Requesting: {} -> {}", alias, qry); - builder.addItem(alias,qry); + readRequestBuilder.addItem(alias, qry); }); - response = builder + plcReadResponse = readRequestBuilder .build() .execute() .get(requestTimeoutMs, TimeUnit.MILLISECONDS); @@ -128,17 +133,20 @@ public class ScraperTaskImpl implements ScraperTask { handleException(e); return; } - // Add statistics + + LOGGER.debug("Performing statistics"); + // Add some statistics stopWatch.stop(); latencyStatistics.addValue(stopWatch.getNanoTime()); failedStatistics.addValue(0.0); successCounter.incrementAndGet(); // Validate response - validateResponse(response); + validateResponse(plcReadResponse); + // Handle response (Async) - CompletableFuture.runAsync(() -> resultHandler.handle(jobName, connectionAlias, transformResponseToMap(response)), handlerService); + CompletableFuture.runAsync(() -> resultHandler.handle(jobName, connectionAlias, transformResponseToMap(plcReadResponse)), handlerService); } catch (Exception e) { - LOGGER.debug("Exception during scrape", e); + LOGGER.warn("Exception during scraping of Job {}, Connection-Alias {}: Error-message: {} - for stack-trace change logging to DEBUG", jobName,connectionAlias,e.getMessage()); handleException(e); } finally { if (connection != null) { @@ -151,6 +159,10 @@ public class ScraperTaskImpl implements ScraperTask { } } + /** + * validate read response due to failed fields + * @param response acquired response + */ private void validateResponse(PlcReadResponse response) { Map<String, PlcResponseCode> failedFields = response.getFieldNames().stream() .filter(name -> !PlcResponseCode.OK.equals(response.getResponseCode(name))) @@ -163,6 +175,11 @@ public class ScraperTaskImpl implements ScraperTask { } } + /** + * transforms the read-response to a Map of String (Key) and Object(Value) + * @param response response from PLC + * @return transformed Map + */ private Map<String, Object> transformResponseToMap(PlcReadResponse response) { return response.getFieldNames().stream() .collect(Collectors.toMap( @@ -203,7 +220,9 @@ public class ScraperTaskImpl implements ScraperTask { @Override public void handleException(Exception e) { - LOGGER.debug("Exception: ", e); + if(LOGGER.isDebugEnabled()) { + LOGGER.debug("Detailed exception occurred at scraping", e); + } failedStatistics.addValue(1.0); } diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java index 98b5b79..9c84b0b 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfiguration.java @@ -23,11 +23,16 @@ import java.util.List; import java.util.Map; /** - * Created by timbo on 2019-03-05 + * basic interface for the configuration of a scrape job */ public interface JobConfiguration { String getName(); + String getTriggerConfig(); + + @Deprecated + Integer getScrapeRate(); + List<String> getSources(); Map<String, String> getFields(); diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationClassicImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationClassicImpl.java new file mode 100644 index 0000000..6688363 --- /dev/null +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationClassicImpl.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.plc4x.java.scraper.config; + +import org.apache.plc4x.java.scraper.ScrapeJobImpl; +import org.apache.plc4x.java.scraper.config.triggeredscraper.JobConfigurationTriggeredImplBuilder; + +import java.util.List; +import java.util.Map; + +/** + * Configuration for one {@link ScrapeJobImpl} in the @{@link ScraperConfigurationClassicImpl}. + * + * @deprecated Scraper is deprecated please use {@link JobConfigurationTriggeredImplBuilder} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} + */ +@Deprecated +public class JobConfigurationClassicImpl extends JobConfigurationImpl { + + + /** + * Default constructor + * + * @param name Job Name / identifier + * @param triggerConfig configuration string for triggered jobs + * @param scrapeRate + * @param sources source alias (<b>not</b> connection string but the alias (from @{@link ScraperConfigurationClassicImpl}). + * @param fields Map from field alias (how it is named in the result map) to plc4x field query + */ + public JobConfigurationClassicImpl(String name, String triggerConfig, Integer scrapeRate, List<String> sources, Map<String, String> fields) { + super(name, triggerConfig, scrapeRate, sources, fields); + } +} diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationImplBuilder.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationClassicImplBuilder.java similarity index 62% rename from plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationImplBuilder.java rename to plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationClassicImplBuilder.java index 418d2fb..bbbcb2a 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationImplBuilder.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationClassicImplBuilder.java @@ -19,45 +19,47 @@ package org.apache.plc4x.java.scraper.config; +import org.apache.plc4x.java.scraper.config.triggeredscraper.JobConfigurationTriggeredImpl; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; /** - * @deprecated Scraper is deprecated please use {@link org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredJobConfiguration} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} + * @deprecated Scraper is deprecated please use {@link JobConfigurationTriggeredImpl} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} */ @Deprecated -public class JobConfigurationImplBuilder { +public class JobConfigurationClassicImplBuilder { - private final ScraperConfigurationBuilder parent; + private final ScraperConfigurationClassicImplBuilder parent; private final String name; private final int scrapeRateMs; private final List<String> sources = new ArrayList<>(); private final Map<String, String> fields = new HashMap<>(); - public JobConfigurationImplBuilder(ScraperConfigurationBuilder parent, String name, int scrapeRateMs) { + public JobConfigurationClassicImplBuilder(ScraperConfigurationClassicImplBuilder parent, String name, int scrapeRateMs) { this.parent = parent; this.name = name; this.scrapeRateMs = scrapeRateMs; } - public JobConfigurationImplBuilder source(String alias) { + public JobConfigurationClassicImplBuilder source(String alias) { this.sources.add(alias); return this; } - public JobConfigurationImplBuilder field(String alias, String fieldQuery) { + public JobConfigurationClassicImplBuilder field(String alias, String fieldQuery) { this.fields.put(alias, fieldQuery); return this; } - private JobConfigurationImpl buildInternal() { - return new JobConfigurationImpl(name, scrapeRateMs, sources, fields); + private JobConfigurationClassicImpl buildInternal() { + return new JobConfigurationClassicImpl(name,null, scrapeRateMs, sources, fields); } - public ScraperConfigurationBuilder build() { + public ScraperConfigurationClassicImplBuilder build() { parent.addJobConfiguration(this.buildInternal()); return this.parent; } diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationImpl.java index 044a93b..81438b9 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationImpl.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/JobConfigurationImpl.java @@ -21,37 +21,36 @@ package org.apache.plc4x.java.scraper.config; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.plc4x.java.scraper.ScrapeJobImpl; +import org.apache.plc4x.java.api.exceptions.PlcNotImplementedException; import java.util.List; import java.util.Map; /** - * Configuration for one {@link ScrapeJobImpl} in the @{@link ScraperConfiguration}. - * - * @deprecated Scraper is deprecated please use {@link org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredJobConfigurationBuilder} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} + * abstract configuration for scrape-job configuration */ -@Deprecated public class JobConfigurationImpl implements JobConfiguration { - - private final String name; - private final int scrapeRate; - private final List<String> sources; - private final Map<String, String> fields; + protected final String name; + protected final String triggerConfig; + protected final Integer scrapeRate; + protected final List<String> sources; + protected final Map<String, String> fields; /** * Default constructor * @param name Job Name / identifier - * @param scrapeRate Scrape rate in ms - * @param sources source alias (<b>not</b> connection string but the alias (from @{@link ScraperConfiguration}). + * @param triggerConfig configuration string for triggered jobs + * @param sources source alias (<b>not</b> connection string but the alias (from @{@link ScraperConfigurationClassicImpl}). * @param fields Map from field alias (how it is named in the result map) to plc4x field query */ @JsonCreator public JobConfigurationImpl(@JsonProperty(value = "name", required = true) String name, - @JsonProperty(value = "scrapeRate", required = true) int scrapeRate, + @JsonProperty(value = "triggerConfig") String triggerConfig, + @JsonProperty(value = "scrapeRate") Integer scrapeRate, @JsonProperty(value = "sources", required = true) List<String> sources, @JsonProperty(value = "fields", required = true) Map<String, String> fields) { this.name = name; + this.triggerConfig = triggerConfig; this.scrapeRate = scrapeRate; this.sources = sources; this.fields = fields; @@ -62,8 +61,9 @@ public class JobConfigurationImpl implements JobConfiguration { return name; } - public int getScrapeRate() { - return scrapeRate; + @Override + public String getTriggerConfig() { + return triggerConfig; } @Override @@ -75,4 +75,9 @@ public class JobConfigurationImpl implements JobConfiguration { public Map<String, String> getFields() { return fields; } + + @Override + public Integer getScrapeRate() { + return scrapeRate; + } } diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java index 9b0804b..c3cb2da 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfiguration.java @@ -19,85 +19,43 @@ package org.apache.plc4x.java.scraper.config; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.MismatchedInputException; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.plc4x.java.scraper.ScrapeJob; -import org.apache.plc4x.java.scraper.ScrapeJobImpl; import org.apache.plc4x.java.scraper.exception.ScraperConfigurationException; import org.apache.plc4x.java.scraper.exception.ScraperException; -import org.apache.plc4x.java.scraper.ScraperImpl; -import org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredJobConfiguration; -import org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScrapeJobImpl; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; /** - * Configuration class for {@link ScraperImpl}. - * - * @deprecated Scraper is deprecated please use {@link org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredScraperConfiguration} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} + * interface for basic configuration of scraper */ -@Deprecated -public class ScraperConfiguration { - - private final Map<String, String> sources; - private final List<JobConfigurationImpl> jobConfigurations; - - /** - * Default constructor. - * - * @param sources Map from connection alias to connection string - * @param jobConfigurations List of configurations one for each Job - */ - @JsonCreator - public ScraperConfiguration(@JsonProperty(value = "sources", required = true) Map<String, String> sources, - @JsonProperty(value = "jobs", required = true) List<JobConfigurationImpl> jobConfigurations) { - checkNoUnreferencedSources(sources, jobConfigurations); - // TODO Warning on too many sources?! - this.sources = sources; - this.jobConfigurations = jobConfigurations; - } - - private void checkNoUnreferencedSources(Map<String, String> sources, List<JobConfigurationImpl> jobConfigurations) { - Set<String> unreferencedSources = jobConfigurations.stream() - .flatMap(job -> job.getSources().stream()) - .filter(source -> !sources.containsKey(source)) - .collect(Collectors.toSet()); - if (!unreferencedSources.isEmpty()) { - throw new ScraperConfigurationException("There are the following unreferenced sources: " + unreferencedSources); - } - } - - public static ScraperConfiguration fromYaml(String yaml) { +public interface ScraperConfiguration { + static <T>T fromYaml(String yaml, Class<T> clazz) { ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); try { - return mapper.readValue(yaml, ScraperConfiguration.class); + return mapper.readValue(yaml, clazz); } catch (IOException e) { throw new ScraperConfigurationException("Unable to parse given yaml configuration!", e); } } - public static ScraperConfiguration fromJson(String json) { + static <T>T fromJson(String json, Class<T> clazz) { ObjectMapper mapper = new ObjectMapper(new JsonFactory()); try { - return mapper.readValue(json, ScraperConfiguration.class); + return mapper.readValue(json, clazz); } catch (IOException e) { throw new ScraperConfigurationException("Unable to parse given json configuration!", e); } } - public static ScraperConfiguration fromFile(String path) throws IOException { + static <T>T fromFile(String path, Class<T> clazz) throws IOException { ObjectMapper mapper; if (path.endsWith("json")) { mapper = new ObjectMapper(new JsonFactory()); @@ -107,7 +65,7 @@ public class ScraperConfiguration { throw new ScraperConfigurationException("Only files with extensions json, yml or yaml can be read"); } try { - return mapper.readValue(new File(path), ScraperConfiguration.class); + return mapper.readValue(new File(path), clazz); } catch (FileNotFoundException e) { throw new ScraperConfigurationException("Unable to find configuration given configuration file at '" + path + "'", e); } catch (MismatchedInputException e) { @@ -115,42 +73,9 @@ public class ScraperConfiguration { } } - public Map<String, String> getSources() { - return sources; - } + Map<String, String> getSources(); - public List<JobConfigurationImpl> getJobConfigurations() { - return jobConfigurations; - } + List<JobConfigurationImpl> getJobConfigurations(); - public List<ScrapeJob> getJobs() throws ScraperException { - List<ScrapeJob> scrapeJobs = new ArrayList<>(); - for(JobConfiguration jobConfiguration:jobConfigurations){ - if(jobConfiguration instanceof JobConfigurationImpl){ - JobConfigurationImpl jobConfigurationImpl = (JobConfigurationImpl)jobConfiguration; - scrapeJobs.add(new ScrapeJobImpl(jobConfiguration.getName(), - jobConfigurationImpl.getScrapeRate(), - getSourcesForAliases(jobConfiguration.getSources()), - jobConfiguration.getFields())); - } - else{ - if(jobConfiguration instanceof TriggeredJobConfiguration){ - TriggeredJobConfiguration triggeredJobConfiguration = (TriggeredJobConfiguration) jobConfiguration; - scrapeJobs.add(new TriggeredScrapeJobImpl(jobConfiguration.getName(), - triggeredJobConfiguration.getTriggerConfig(), - getSourcesForAliases(jobConfiguration.getSources()), - jobConfiguration.getFields())); - } - } - } - return scrapeJobs; - } - - private Map<String, String> getSourcesForAliases(List<String> aliases) { - return aliases.stream() - .collect(Collectors.toMap( - Function.identity(), - sources::get - )); - } + List<ScrapeJob> getJobs() throws ScraperException; } diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationClassicImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationClassicImpl.java new file mode 100644 index 0000000..aafa373 --- /dev/null +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationClassicImpl.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.plc4x.java.scraper.config; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.plc4x.java.scraper.ScrapeJob; +import org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImpl; +import org.apache.plc4x.java.scraper.exception.ScraperConfigurationException; +import org.apache.plc4x.java.scraper.exception.ScraperException; +import org.apache.plc4x.java.scraper.ScraperImpl; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Configuration class for {@link ScraperImpl}. + * + * @deprecated Scraper is deprecated please use {@link ScraperConfigurationTriggeredImpl} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} + */ +@Deprecated +public class ScraperConfigurationClassicImpl implements ScraperConfiguration { + + private final Map<String, String> sources; + private final List<JobConfigurationImpl> jobConfigurations; + + /** + * Default constructor. + * + * @param sources Map from connection alias to connection string + * @param jobConfigurations List of configurations one for each Job + */ + @JsonCreator + public ScraperConfigurationClassicImpl(@JsonProperty(value = "sources", required = true) Map<String, String> sources, + @JsonProperty(value = "jobs", required = true) List<JobConfigurationImpl> jobConfigurations) { + checkNoUnreferencedSources(sources, jobConfigurations); + this.sources = sources; + this.jobConfigurations = jobConfigurations; + } + + private void checkNoUnreferencedSources(Map<String, String> sources, List<JobConfigurationImpl> jobConfigurations) { + Set<String> unreferencedSources = jobConfigurations.stream() + .flatMap(job -> job.getSources().stream()) + .filter(source -> !sources.containsKey(source)) + .collect(Collectors.toSet()); + if (!unreferencedSources.isEmpty()) { + throw new ScraperConfigurationException("There are the following unreferenced sources: " + unreferencedSources); + } + } + + @Override + public Map<String, String> getSources() { + return sources; + } + + @Override + public List<JobConfigurationImpl> getJobConfigurations() { + return jobConfigurations; + } + + @Override + public List<ScrapeJob> getJobs() throws ScraperException { + return ScraperConfigurationTriggeredImpl.getJobs(jobConfigurations,sources); + } + + +} diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationBuilder.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationClassicImplBuilder.java similarity index 61% rename from plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationBuilder.java rename to plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationClassicImplBuilder.java index f0cb3b9..e64a009 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationBuilder.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationClassicImplBuilder.java @@ -19,34 +19,36 @@ package org.apache.plc4x.java.scraper.config; +import org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImplBuilder; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; /** - * @deprecated Scraper is deprecated please use {@link org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredScraperConfigurationBuilder} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} + * @deprecated Scraper is deprecated please use {@link ScraperConfigurationTriggeredImplBuilder} instead all functions are supplied as well see java-doc of {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl} */ @Deprecated -public class ScraperConfigurationBuilder { +public class ScraperConfigurationClassicImplBuilder { private final Map<String, String> sources = new HashMap<>(); private final List<JobConfigurationImpl> jobConfigurations = new ArrayList<>(); - public ScraperConfigurationBuilder addSource(String alias, String connectionString) { + public ScraperConfigurationClassicImplBuilder addSource(String alias, String connectionString) { sources.put(alias, connectionString); return this; } - public JobConfigurationImplBuilder job(String name, int scrapeRateMs) { - return new JobConfigurationImplBuilder(this, name, scrapeRateMs); + public JobConfigurationClassicImplBuilder job(String name, int scrapeRateMs) { + return new JobConfigurationClassicImplBuilder(this, name, scrapeRateMs); } public ScraperConfiguration build() { - return new ScraperConfiguration(sources, jobConfigurations); + return new ScraperConfigurationClassicImpl(sources, jobConfigurations); } - public void addJobConfiguration(JobConfigurationImpl configuration) { + public void addJobConfiguration(JobConfigurationClassicImpl configuration) { this.jobConfigurations.add(configuration); } } diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/JobConfigurationTriggeredImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/JobConfigurationTriggeredImpl.java new file mode 100644 index 0000000..48d850c --- /dev/null +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/JobConfigurationTriggeredImpl.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.plc4x.java.scraper.config.triggeredscraper; + + +import org.apache.plc4x.java.scraper.config.JobConfigurationImpl; +import org.apache.plc4x.java.scraper.config.ScraperConfigurationClassicImpl; + +import java.util.List; +import java.util.Map; + +/** + * Configuration for one {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScrapeJobImpl} in the {@link ScraperConfigurationTriggeredImpl}. + */ +public class JobConfigurationTriggeredImpl extends JobConfigurationImpl { + + + /** + * Default constructor + * + * @param name Job Name / identifier + * @param triggerConfig configuration string for triggered jobs + * @param scrapeRate + * @param sources source alias (<b>not</b> connection string but the alias (from @{@link ScraperConfigurationClassicImpl}). + * @param fields Map from field alias (how it is named in the result map) to plc4x field query + */ + public JobConfigurationTriggeredImpl(String name, String triggerConfig, Integer scrapeRate, List<String> sources, Map<String, String> fields) { + super(name, triggerConfig, scrapeRate, sources, fields); + } +} diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredJobConfigurationBuilder.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/JobConfigurationTriggeredImplBuilder.java similarity index 76% rename from plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredJobConfigurationBuilder.java rename to plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/JobConfigurationTriggeredImplBuilder.java index 817c3c9..48b7bd6 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredJobConfigurationBuilder.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/JobConfigurationTriggeredImplBuilder.java @@ -26,16 +26,16 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -public class TriggeredJobConfigurationBuilder { +public class JobConfigurationTriggeredImplBuilder { - private final TriggeredScraperConfigurationBuilder parent; + private final ScraperConfigurationTriggeredImplBuilder parent; private final String name; private final String triggerConfig; private final List<String> sources = new ArrayList<>(); private final Map<String, String> fields = new HashMap<>(); - public TriggeredJobConfigurationBuilder(TriggeredScraperConfigurationBuilder parent, String name, String triggerConfig) { + public JobConfigurationTriggeredImplBuilder(ScraperConfigurationTriggeredImplBuilder parent, String name, String triggerConfig) { if(parent==null){ throw new ScraperConfigurationException("parent builder cannot be null"); } @@ -47,7 +47,7 @@ public class TriggeredJobConfigurationBuilder { this.triggerConfig = triggerConfig; } - public TriggeredJobConfigurationBuilder source(String alias) { + public JobConfigurationTriggeredImplBuilder source(String alias) { if(alias==null || alias.isEmpty()){ throw new ScraperConfigurationException("source alias cannot be null or empty"); } @@ -55,16 +55,16 @@ public class TriggeredJobConfigurationBuilder { return this; } - public TriggeredJobConfigurationBuilder field(String alias, String fieldQuery) { + public JobConfigurationTriggeredImplBuilder field(String alias, String fieldQuery) { this.fields.put(alias, fieldQuery); return this; } - private TriggeredJobConfiguration buildInternal() { - return new TriggeredJobConfiguration(name, triggerConfig, sources, fields); + private JobConfigurationTriggeredImpl buildInternal() { + return new JobConfigurationTriggeredImpl(name, triggerConfig,null, sources, fields); } - public TriggeredScraperConfigurationBuilder build() { + public ScraperConfigurationTriggeredImplBuilder build() { parent.addJobConfiguration(this.buildInternal()); return this.parent; } diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredScraperConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/ScraperConfigurationTriggeredImpl.java similarity index 50% rename from plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredScraperConfiguration.java rename to plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/ScraperConfigurationTriggeredImpl.java index 40e66ee..af552a3 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredScraperConfiguration.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/ScraperConfigurationTriggeredImpl.java @@ -21,21 +21,18 @@ package org.apache.plc4x.java.scraper.config.triggeredscraper; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.exc.MismatchedInputException; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.plc4x.java.scraper.ScrapeJob; import org.apache.plc4x.java.scraper.ScrapeJobImpl; +import org.apache.plc4x.java.scraper.config.JobConfigurationImpl; +import org.apache.plc4x.java.scraper.config.ScraperConfiguration; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.apache.plc4x.java.scraper.config.JobConfiguration; -import org.apache.plc4x.java.scraper.config.JobConfigurationImpl; +import org.apache.plc4x.java.scraper.config.JobConfigurationClassicImpl; import org.apache.plc4x.java.scraper.exception.ScraperConfigurationException; import org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScrapeJobImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -46,10 +43,11 @@ import java.util.stream.Collectors; /** * Configuration class for {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl}. */ -public class TriggeredScraperConfiguration { +public class ScraperConfigurationTriggeredImpl implements ScraperConfiguration { + private static final Logger logger = LoggerFactory.getLogger( ScraperConfigurationTriggeredImpl.class ); private final Map<String, String> sources; - private final List<TriggeredJobConfiguration> jobConfigurations; + private final List<JobConfigurationImpl> jobConfigurations; /** * Default constructor. @@ -58,14 +56,14 @@ public class TriggeredScraperConfiguration { * @param jobConfigurations List of configurations one for each Job */ @JsonCreator - public TriggeredScraperConfiguration(@JsonProperty(value = "sources", required = true) Map<String, String> sources, - @JsonProperty(value = "jobs", required = true) List<TriggeredJobConfiguration> jobConfigurations) { + public ScraperConfigurationTriggeredImpl(@JsonProperty(value = "sources", required = true) Map<String, String> sources, + @JsonProperty(value = "jobs", required = true) List<JobConfigurationImpl> jobConfigurations) { checkNoUnreferencedSources(sources, jobConfigurations); this.sources = sources; this.jobConfigurations = jobConfigurations; } - private void checkNoUnreferencedSources(Map<String, String> sources, List<TriggeredJobConfiguration> jobConfigurations) { + private void checkNoUnreferencedSources(Map<String, String> sources, List<JobConfigurationImpl> jobConfigurations) { Set<String> unreferencedSources = jobConfigurations.stream() .flatMap(job -> job.getSources().stream()) .filter(source -> !sources.containsKey(source)) @@ -75,74 +73,51 @@ public class TriggeredScraperConfiguration { } } - public static TriggeredScraperConfiguration fromYaml(String yaml) { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); - try { - return mapper.readValue(yaml, TriggeredScraperConfiguration.class); - } catch (IOException e) { - throw new ScraperConfigurationException("Unable to parse given yaml configuration!", e); - } - } - - public static TriggeredScraperConfiguration fromJson(String json) { - ObjectMapper mapper = new ObjectMapper(new JsonFactory()); - try { - return mapper.readValue(json, TriggeredScraperConfiguration.class); - } catch (IOException e) { - throw new ScraperConfigurationException("Unable to parse given json configuration!", e); - } - } - - public static TriggeredScraperConfiguration fromFile(String path) throws IOException { - ObjectMapper mapper; - if (path.endsWith("json")) { - mapper = new ObjectMapper(new JsonFactory()); - } else if (path.endsWith("yml") || path.endsWith("yaml")) { - mapper = new ObjectMapper(new YAMLFactory()); - } else { - throw new ScraperConfigurationException("Only files with extensions json, yml or yaml can be read"); - } - try { - return mapper.readValue(new File(path), TriggeredScraperConfiguration.class); - } catch (FileNotFoundException e) { - throw new ScraperConfigurationException("Unable to find configuration given configuration file at '" + path + "'", e); - } catch (MismatchedInputException e) { - throw new ScraperConfigurationException("Given configuration is in wrong format!", e); - } - } - + @Override public Map<String, String> getSources() { return sources; } - public List<TriggeredJobConfiguration> getJobConfigurations() { + @Override + public List<JobConfigurationImpl> getJobConfigurations() { return jobConfigurations; } + @Override public List<ScrapeJob> getJobs() throws ScraperException { + return getJobs(jobConfigurations,sources); + } + + public static List<ScrapeJob> getJobs(List<JobConfigurationImpl> jobConfigurations, Map<String, String> sources) throws ScraperConfigurationException { List<ScrapeJob> scrapeJobs = new ArrayList<>(); for(JobConfiguration jobConfiguration:jobConfigurations){ - if(jobConfiguration instanceof TriggeredJobConfiguration){ - JobConfigurationImpl jobConfigurationImpl = (JobConfigurationImpl)jobConfiguration; - scrapeJobs.add(new ScrapeJobImpl(jobConfiguration.getName(), - jobConfigurationImpl.getScrapeRate(), - getSourcesForAliases(jobConfiguration.getSources()), + if(jobConfiguration.getTriggerConfig()!=null){ + logger.info("Assuming job as triggered job because triggerConfig has been set"); + scrapeJobs.add(new TriggeredScrapeJobImpl(jobConfiguration.getName(), + jobConfiguration.getTriggerConfig(), + getSourcesForAliases(jobConfiguration.getSources(),sources), jobConfiguration.getFields())); } - else{ - if(jobConfiguration instanceof TriggeredJobConfiguration){ - TriggeredJobConfiguration triggeredJobConfiguration = (TriggeredJobConfiguration) jobConfiguration; - scrapeJobs.add(new TriggeredScrapeJobImpl(jobConfiguration.getName(), - triggeredJobConfiguration.getTriggerConfig(), - getSourcesForAliases(jobConfiguration.getSources()), + else { + if(jobConfiguration.getScrapeRate()!=null){ + logger.info("Assuming job as classic job because triggerConfig has NOT been set but scrapeRate has."); + scrapeJobs.add(new ScrapeJobImpl( + jobConfiguration.getName(), + jobConfiguration.getScrapeRate(), + getSourcesForAliases(jobConfiguration.getSources(),sources), jobConfiguration.getFields())); } + else { + logger.info("Job has lack of trigger/scheduled config"); + throw new ScraperConfigurationException( + String.format("Job %s was intended to be o triggered annotation, but no triggerConfig-Field could be found. Canceling!",jobConfiguration.getName())); + } } } return scrapeJobs; } - private Map<String, String> getSourcesForAliases(List<String> aliases) { + private static Map<String, String> getSourcesForAliases(List<String> aliases, Map<String, String> sources) { return aliases.stream() .collect(Collectors.toMap( Function.identity(), diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredScraperConfigurationBuilder.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/ScraperConfigurationTriggeredImplBuilder.java similarity index 62% rename from plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredScraperConfigurationBuilder.java rename to plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/ScraperConfigurationTriggeredImplBuilder.java index da02027..6c1dc64 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredScraperConfigurationBuilder.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/ScraperConfigurationTriggeredImplBuilder.java @@ -19,30 +19,32 @@ package org.apache.plc4x.java.scraper.config.triggeredscraper; +import org.apache.plc4x.java.scraper.config.JobConfigurationImpl; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -public class TriggeredScraperConfigurationBuilder { +public class ScraperConfigurationTriggeredImplBuilder { private final Map<String, String> sources = new HashMap<>(); - private final List<TriggeredJobConfiguration> jobConfigurations = new ArrayList<>(); + private final List<JobConfigurationImpl> jobConfigurations = new ArrayList<>(); - public TriggeredScraperConfigurationBuilder addSource(String alias, String connectionString) { + public ScraperConfigurationTriggeredImplBuilder addSource(String alias, String connectionString) { sources.put(alias, connectionString); return this; } - public TriggeredJobConfigurationBuilder job(String name, String triggerConfig) { - return new TriggeredJobConfigurationBuilder(this, name, triggerConfig); + public JobConfigurationTriggeredImplBuilder job(String name, String triggerConfig) { + return new JobConfigurationTriggeredImplBuilder(this, name, triggerConfig); } - public TriggeredScraperConfiguration build() { - return new TriggeredScraperConfiguration(sources, jobConfigurations); + public ScraperConfigurationTriggeredImpl build() { + return new ScraperConfigurationTriggeredImpl(sources, jobConfigurations); } - public void addJobConfiguration(TriggeredJobConfiguration configuration) { + public void addJobConfiguration(JobConfigurationTriggeredImpl configuration) { this.jobConfigurations.add(configuration); } } diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredJobConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredJobConfiguration.java deleted file mode 100644 index 73bb20e..0000000 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/config/triggeredscraper/TriggeredJobConfiguration.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.plc4x.java.scraper.config.triggeredscraper; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import org.apache.plc4x.java.scraper.config.JobConfiguration; -import org.apache.plc4x.java.scraper.config.ScraperConfiguration; - -import java.util.List; -import java.util.Map; - -/** - * Configuration for one {@link org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScrapeJobImpl} in the {@link org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredScraperConfiguration}. - */ -public class TriggeredJobConfiguration implements JobConfiguration { - - private final String name; - private final String triggerConfig; - private final List<String> sources; - private final Map<String, String> fields; - - /** - * Default constructor - * @param name Job Name / identifier - * @param triggerConfig configuration string for triggered jobs - * @param sources source alias (<b>not</b> connection string but the alias (from @{@link ScraperConfiguration}). - * @param fields Map from field alias (how it is named in the result map) to plc4x field query - */ - @JsonCreator - public TriggeredJobConfiguration(@JsonProperty(value = "name", required = true) String name, - @JsonProperty(value = "triggerConfig", required = true) String triggerConfig, - @JsonProperty(value = "sources", required = true) List<String> sources, - @JsonProperty(value = "fields", required = true) Map<String, String> fields) { - this.name = name; - this.triggerConfig = triggerConfig; - this.sources = sources; - this.fields = fields; - } - - public String getName() { - return name; - } - - public String getTriggerConfig() { - return triggerConfig; - } - - public List<String> getSources() { - return sources; - } - - public Map<String, String> getFields() { - return fields; - } -} diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScrapeJobImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScrapeJobImpl.java index 1a73144..b9f50cc 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScrapeJobImpl.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScrapeJobImpl.java @@ -20,6 +20,7 @@ package org.apache.plc4x.java.scraper.triggeredscraper; import org.apache.plc4x.java.scraper.ScrapeJob; +import org.apache.plc4x.java.scraper.exception.ScraperConfigurationException; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.TriggerConfiguration; @@ -33,7 +34,7 @@ public class TriggeredScrapeJobImpl implements ScrapeJob { private final TriggerConfiguration triggerConfiguration; - public TriggeredScrapeJobImpl(String jobName, String triggerConfig, Map<String, String> connections, Map<String, String> fields) throws ScraperException { + public TriggeredScrapeJobImpl(String jobName, String triggerConfig, Map<String, String> connections, Map<String, String> fields) throws ScraperConfigurationException { this.jobName = jobName; this.triggerConfig = triggerConfig; this.sourceConnections = connections; diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperImpl.java index d3e8a19..386a412 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperImpl.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperImpl.java @@ -32,8 +32,9 @@ import org.apache.plc4x.java.api.exceptions.PlcConnectionException; import org.apache.plc4x.java.api.exceptions.PlcRuntimeException; import org.apache.plc4x.java.api.messages.PlcReadResponse; import org.apache.plc4x.java.scraper.*; +import org.apache.plc4x.java.scraper.config.ScraperConfiguration; import org.apache.plc4x.java.scraper.exception.ScraperException; -import org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredScraperConfiguration; +import org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImpl; import org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector.TriggerCollector; import org.apache.plc4x.java.scraper.util.PercentageAboveThreshold; import org.apache.plc4x.java.spi.PlcDriver; @@ -64,22 +65,12 @@ import java.util.stream.Collectors; public class TriggeredScraperImpl implements Scraper, TriggeredScraperMBean { private static final Logger LOGGER = LoggerFactory.getLogger(TriggeredScraperImpl.class); - public static final String MX_DOMAIN = "org.apache.plc4x.java"; + private static final String MX_DOMAIN = "org.apache.plc4x.java"; private static final int DEFAULT_FUTURE_TIME_OUT = 2000; - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(10, - new BasicThreadFactory.Builder() - .namingPattern("triggeredscraper-scheduling-thread-%d") - .daemon(false) - .build() - ); - private final ExecutorService executorService = Executors.newFixedThreadPool(4, - new BasicThreadFactory.Builder() - .namingPattern("triggeredscraper-executer-thread-%d") - .daemon(true) - .build() - ); + private final ScheduledExecutorService scheduler; + private final ExecutorService executorService; private final ResultHandler resultHandler; @@ -99,7 +90,7 @@ public class TriggeredScraperImpl implements Scraper, TriggeredScraperMBean { * @param config Configuration to use. * @param resultHandler handler the defines the processing of acquired data */ - public TriggeredScraperImpl(TriggeredScraperConfiguration config, ResultHandler resultHandler,TriggerCollector triggerCollector) throws ScraperException { + public TriggeredScraperImpl(ScraperConfiguration config, ResultHandler resultHandler, TriggerCollector triggerCollector) throws ScraperException { this(resultHandler, createPooledDriverManager(), config.getJobs(),triggerCollector,DEFAULT_FUTURE_TIME_OUT); } @@ -109,12 +100,22 @@ public class TriggeredScraperImpl implements Scraper, TriggeredScraperMBean { * @param plcDriverManager external DriverManager * @param resultHandler handler the defines the processing of acquired data */ - public TriggeredScraperImpl(TriggeredScraperConfiguration config, PlcDriverManager plcDriverManager, ResultHandler resultHandler,TriggerCollector triggerCollector) throws ScraperException { + public TriggeredScraperImpl(ScraperConfiguration config, PlcDriverManager plcDriverManager, ResultHandler resultHandler,TriggerCollector triggerCollector) throws ScraperException { this(resultHandler, plcDriverManager, config.getJobs(),triggerCollector,DEFAULT_FUTURE_TIME_OUT); } /** * Creates a Scraper instance from a configuration. + * @param config Configuration to use. + * @param plcDriverManager external DriverManager + * @param resultHandler handler the defines the processing of acquired data + */ + public TriggeredScraperImpl(ScraperConfigurationTriggeredImpl config, PlcDriverManager plcDriverManager, ResultHandler resultHandler, TriggerCollector triggerCollector, int poolSizeScheduler, int poolSizeExecutor) throws ScraperException { + this(resultHandler, plcDriverManager, config.getJobs(),triggerCollector,DEFAULT_FUTURE_TIME_OUT,poolSizeScheduler,poolSizeExecutor); + } + + /** + * Creates a Scraper instance from a configuration. * @param plcDriverManager external DriverManager * @param resultHandler handler the defines the processing of acquired data * @param jobs list of jobs that scraper shall handle @@ -122,12 +123,32 @@ public class TriggeredScraperImpl implements Scraper, TriggeredScraperMBean { * @param futureTimeOut max duration of future to return a result */ public TriggeredScraperImpl(ResultHandler resultHandler, PlcDriverManager plcDriverManager, List<ScrapeJob> jobs,TriggerCollector triggerCollector, long futureTimeOut) { + this(resultHandler,plcDriverManager,jobs,triggerCollector,futureTimeOut,20,5); + } + + public TriggeredScraperImpl(ResultHandler resultHandler, PlcDriverManager plcDriverManager, List<ScrapeJob> jobs,TriggerCollector triggerCollector, long futureTimeOut, int poolSizeScheduler, int poolSizeExecutor) { this.resultHandler = resultHandler; Validate.notEmpty(jobs); this.driverManager = plcDriverManager; this.jobs = jobs; this.triggerCollector = triggerCollector; this.futureTimeOut = futureTimeOut; + + this.scheduler = Executors.newScheduledThreadPool(poolSizeScheduler, + new BasicThreadFactory.Builder() + .namingPattern("triggeredscraper-scheduling-thread-%d") + .daemon(false) + .build() + ); + + this.executorService = Executors.newFixedThreadPool(poolSizeExecutor, + new BasicThreadFactory.Builder() + .namingPattern("triggeredscraper-executor-thread-%d") + .daemon(true) + .build() + ); + + // Register MBean mBeanServer = ManagementFactory.getPlatformMBeanServer(); try { diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperTask.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperTask.java index 92cfcd9..ca18021 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperTask.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperTask.java @@ -46,7 +46,6 @@ import java.util.stream.Collectors; /** * performs the triggered task from a job for one device based on the TriggerHandler as defined in Configuration - * ToDo Implement the monitoring as well: PLC4X-90 */ public class TriggeredScraperTask implements ScraperTask, TriggeredScraperTaskMBean { private static final Logger LOGGER = LoggerFactory.getLogger(TriggeredScraperTask.class); @@ -89,7 +88,6 @@ public class TriggeredScraperTask implements ScraperTask, TriggeredScraperTaskMB } @Override - //ToDo code-refactoring and improved testing --> PLC4X-90 public void run() { if(LOGGER.isTraceEnabled()) { LOGGER.trace("Check condition for task of job {} for connection {}", jobName, connectionAlias); @@ -116,14 +114,18 @@ public class TriggeredScraperTask implements ScraperTask, TriggeredScraperTaskMB if(LOGGER.isTraceEnabled()) { LOGGER.trace("Connection to {} established: {}", connectionString, connection); } - PlcReadResponse response; + + PlcReadResponse plcReadResponse; try { - PlcReadRequest.Builder builder = connection.readRequestBuilder(); - fields.forEach((alias, qry) -> { - LOGGER.trace("Requesting: {} -> {}", alias, qry); - builder.addItem(alias, qry); - }); - response = builder + PlcReadRequest.Builder readRequestBuilder = connection.readRequestBuilder(); + for(Map.Entry<String,String> entry:fields.entrySet()){ + if(LOGGER.isTraceEnabled()) { + LOGGER.trace("Requesting: {} -> {}", entry.getKey(), entry.getValue()); + } + readRequestBuilder.addItem(entry.getKey(),entry.getValue()); + } + //build and send request and store result in read response + plcReadResponse = readRequestBuilder .build() .execute() .get(requestTimeoutMs, TimeUnit.MILLISECONDS); @@ -132,19 +134,19 @@ public class TriggeredScraperTask implements ScraperTask, TriggeredScraperTaskMB handleException(e); return; } + // Add statistics + LOGGER.debug("Performing statistics"); stopWatch.stop(); latencyStatistics.addValue(stopWatch.getNanoTime()); failedStatistics.addValue(0.0); successCounter.incrementAndGet(); // Validate response - validateResponse(response); + validateResponse(plcReadResponse); // Handle response (Async) - CompletableFuture.runAsync(() -> resultHandler.handle(jobName, connectionAlias, TriggeredScraperImpl.convertPlcResponseToMap(response)), executorService); + CompletableFuture.runAsync(() -> resultHandler.handle(jobName, connectionAlias, TriggeredScraperImpl.convertPlcResponseToMap(plcReadResponse)), executorService); } catch (Exception e) { - if(LOGGER.isDebugEnabled()) { - LOGGER.debug("Exception during scrape", e); - } + LOGGER.warn("Exception during scraping of Job {}, Connection-Alias {}: Error-message: {} - for stack-trace change logging to DEBUG", jobName,connectionAlias,e.getMessage()); handleException(e); } finally { if (connection != null) { @@ -175,8 +177,6 @@ public class TriggeredScraperTask implements ScraperTask, TriggeredScraperTaskMB } } - - @Override public String getJobName() { return this.jobName; @@ -204,12 +204,15 @@ public class TriggeredScraperTask implements ScraperTask, TriggeredScraperTaskMB @Override public void handleException(Exception e) { - // TODO Implement this + if(LOGGER.isDebugEnabled()) { + LOGGER.debug("Detailed exception occurred at scraping", e); + } + failedStatistics.addValue(1.0); } @Override public void handleErrorResponse(Map<String, PlcResponseCode> failed) { - // TODO Implement this + LOGGER.warn("Handling error responses: {}", failed); } public PlcDriverManager getDriverManager() { diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/TriggerConfiguration.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/TriggerConfiguration.java index 6fbd234..0049e55 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/TriggerConfiguration.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/TriggerConfiguration.java @@ -19,9 +19,9 @@ package org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler; -import org.apache.plc4x.java.api.exceptions.PlcInvalidFieldException; import org.apache.plc4x.java.api.model.PlcField; import org.apache.plc4x.java.s7.model.S7Field; +import org.apache.plc4x.java.scraper.exception.ScraperConfigurationException; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScrapeJobImpl; import org.slf4j.Logger; @@ -35,28 +35,23 @@ import java.util.regex.Pattern; /** * basic configuration for all available triggers and handling of regarding condition */ -//ToDo: Improve structure to make it more generic --> PLC4X-89 public class TriggerConfiguration{ private static final Logger logger = LoggerFactory.getLogger(TriggerConfiguration.class); private static final String S_7_TRIGGER_VAR = "S7_TRIGGER_VAR"; - private static final String SCHEDULED = "SCHEDULED"; + private static final String SCHEDULED = "SCHEDULED"; + private static final String PREVIOUS_DEF = "PREV"; private static final double TOLERANCE_FLOATING_EQUALITY = 1e-6; private static final Pattern TRIGGER_STRATEGY_PATTERN = Pattern.compile("\\((?<strategy>[A-Z_0-9]+),(?<scheduledInterval>\\d+)(,(\\((?<triggerVar>[^!=<>()]+)\\))((?<comp>[!=<>]{1,2}))(\\((?<compVar>[PREVa-z0-9.\\-]+)\\))((?<concatConn>[ANDOR]{2,3})(\\((?<triggerVar2>[^!=<>()]+)\\))((?<comp2>[!=<>]{1,2}))(\\((?<compVar2>[PREVa-z0-9.\\-]+)\\)))?)?\\)"); - private static final String PREVIOUS_DEF = "PREV"; private final TriggerType triggerType; private final Long scrapeInterval; private TriggeredScrapeJobImpl triggeredScrapeJobImpl; - - private List<TriggerElement> triggerElementList; - - /** * default constructor when an S7Field should be used for triggering * @param triggerType type of trigger from enum @@ -69,7 +64,7 @@ public class TriggerConfiguration{ String scrapeInterval, List<TriggerElement> triggerElementList, TriggeredScrapeJobImpl triggeredScrapeJobImpl) - throws ScraperException { + throws ScraperConfigurationException { this.triggerElementList = triggerElementList; this.triggerType = triggerType; this.triggeredScrapeJobImpl = triggeredScrapeJobImpl; @@ -79,18 +74,17 @@ public class TriggerConfiguration{ if(this.triggerType.equals(TriggerType.S7_TRIGGER_VAR)) { //test for valid field-connection string, on exception quit job and return message to user - - if(this.triggerElementList.isEmpty()){ - exceptionMessage = String.format("No items in trigger List for trigger-type S7_TRIGGER_VAR for Job !", triggeredScrapeJobImpl.getJobName()); - throw new ScraperException(exceptionMessage); - } - checkTriggerVarList(); + if(this.triggerElementList.isEmpty()){ + exceptionMessage = String.format("No items in trigger List for trigger-type S7_TRIGGER_VAR for Job %s!", triggeredScrapeJobImpl.getJobName()); + throw new ScraperConfigurationException(exceptionMessage); + } + checkTriggerVarList(); //ToDo add more and other trigger } else{ exceptionMessage = String.format("TriggerType %s is not yet implemented", this.triggerType); - throw new ScraperException(exceptionMessage); + throw new ScraperConfigurationException(exceptionMessage); } } @@ -99,9 +93,9 @@ public class TriggerConfiguration{ * default constructor when scheduled trigger shall be performed * @param triggerType type of trigger from enum * @param scrapeInterval scrape interval of data from block - * @throws ScraperException when something goes wrong with configuration + * @throws ScraperConfigurationException when something goes wrong with configuration */ - public TriggerConfiguration(TriggerType triggerType, String scrapeInterval) throws ScraperException { + public TriggerConfiguration(TriggerType triggerType, String scrapeInterval) throws ScraperConfigurationException { this.triggerType = triggerType; this.scrapeInterval = parseScrapeInterval(scrapeInterval); this.triggerElementList = new ArrayList<>(); @@ -109,13 +103,13 @@ public class TriggerConfiguration{ /** * checks the trigger list for correct syntax - * @throws ScraperException if syntax isn't correct an exception is thrown + * @throws ScraperConfigurationException if syntax isn't correct an exception is thrown */ - private void checkTriggerVarList() throws ScraperException { + private void checkTriggerVarList() throws ScraperConfigurationException { boolean first = true; for(TriggerElement triggerElement:this.triggerElementList){ if(!first && triggerElement.getConcatType()==null){ - throw new ScraperException("A concat for the second and following trigger must be given!"); + throw new ScraperConfigurationException("A concat for the second and following trigger must be given!"); } first = false; } @@ -125,16 +119,16 @@ public class TriggerConfiguration{ * parses String of scrape interval * @param scrapeInterval string extracted from RegEx * @return converted value - * @throws ScraperException if parsing could not be performed + * @throws ScraperConfigurationException if parsing could not be performed */ - private long parseScrapeInterval(String scrapeInterval) throws ScraperException { + private long parseScrapeInterval(String scrapeInterval) throws ScraperConfigurationException { try { return Long.parseLong(scrapeInterval); } catch (Exception e){ handleException(e); String exceptionMessage = String.format("No valid numeric for scrapeInterval for Job %s: %s",triggeredScrapeJobImpl.getJobName(),scrapeInterval); - throw new ScraperException(exceptionMessage); + throw new ScraperConfigurationException(exceptionMessage); } } @@ -156,7 +150,7 @@ public class TriggerConfiguration{ * @throws ScraperException when an unsupported S7-Type is chosen,which is not (yet) implemented for comparison * ToDo check how to handle time-variables if needed */ - private static Class<?> validateDataType(PlcField plcField) throws ScraperException { + private static Class<?> validateDataType(PlcField plcField) throws ScraperConfigurationException { if(plcField!=null){ Class<?> javaDataType = plcField.getDefaultJavaType(); if(!javaDataType.equals(Boolean.class) @@ -165,18 +159,17 @@ public class TriggerConfiguration{ && !javaDataType.equals(Double.class) ){ String exceptionMessage = String.format("Unsupported plc-trigger variable %s with converted data-type %s used",plcField,plcField.getDefaultJavaType()); - throw new ScraperException(exceptionMessage); + throw new ScraperConfigurationException(exceptionMessage); } return javaDataType; } else{ String exceptionMessage = String.format("Unsupported plc-trigger variable %s with converted data-type %s used",plcField,plcField.getDefaultJavaType()); - throw new ScraperException(exceptionMessage); + throw new ScraperConfigurationException(exceptionMessage); } } - /** * nested class performing the trigger evaluation */ @@ -191,9 +184,9 @@ public class TriggerConfiguration{ /** * does the evaluation of the trigger conditions are met - * //ToDo refactor this + * //ToDo refactor this to improve readability * @return true if trigger conditions are met, false otherwise - * @throws ScraperException + * @throws ScraperException if something went wrong */ boolean evaluateTrigger() throws ScraperException { List<Boolean> triggerResultList = new ArrayList<>(); @@ -204,10 +197,12 @@ public class TriggerConfiguration{ } logger.trace("eval values for job {} and {}: {}",triggeredScrapeJobImpl.getJobName(),connString,acquiredValuesList); } + //iterate through all items of acquirement-list for(int countElements=0; countElements<acquiredValuesList.size();countElements++){ TriggerElement triggerElement = triggerElementList.get(countElements); Object acquiredObject = acquiredValuesList.get(countElements); if(validateDataType(triggerElement.getPlcField()).equals(Boolean.class)){ + //if given type is Boolean boolean currentValue; boolean refValue; try{ @@ -228,7 +223,8 @@ public class TriggerConfiguration{ if(validateDataType(triggerElement.getPlcField()).equals(Double.class) || validateDataType(triggerElement.getPlcField()).equals(Integer.class) || validateDataType(triggerElement.getPlcField()).equals(Long.class)) { - boolean skipComparison = false; + //if given type is numerical + boolean skipComparison = false; //comparison shall be skipped if previous values was null double currentValue; double refValue = 0; try{ @@ -324,6 +320,7 @@ public class TriggerConfiguration{ } return false; } + //check if there is more then one condition for trigger if(triggerResultList.size()>1) { if(logger.isTraceEnabled()) { logger.trace("{}", triggerResultList); @@ -343,13 +340,13 @@ public class TriggerConfiguration{ } } if(combinedResult) { - triggerElementList.forEach(triggerElement -> triggerElement.overrideCompareValue()); + triggerElementList.forEach(TriggerElement::overrideCompareValue); } return combinedResult; } else{ if(triggerResultList.get(0)) { - triggerElementList.forEach(triggerElement -> triggerElement.overrideCompareValue()); + triggerElementList.forEach(TriggerElement::overrideCompareValue); } //return first result because its the only one return triggerResultList.get(0); @@ -364,7 +361,7 @@ public class TriggerConfiguration{ * @return created TriggerConfiguration * @throws ScraperException when something goes wrong */ - public static TriggerConfiguration createConfiguration(String jobTriggerStrategy,TriggeredScrapeJobImpl triggeredScrapeJob) throws ScraperException { + public static TriggerConfiguration createConfiguration(String jobTriggerStrategy,TriggeredScrapeJobImpl triggeredScrapeJob) throws ScraperConfigurationException { Matcher matcher = TRIGGER_STRATEGY_PATTERN.matcher(jobTriggerStrategy); if(matcher.matches()){ @@ -382,7 +379,7 @@ public class TriggerConfiguration{ case S_7_TRIGGER_VAR: if(triggerVar ==null || comparatorString==null || comparatorVariable==null){ - throw new ScraperException("S7_TRIGGER_VAR trigger strategy needs the trigger-condition - information missing! given configString: "+jobTriggerStrategy); + throw new ScraperConfigurationException("S7_TRIGGER_VAR trigger strategy needs the trigger-condition - information missing! given configString: "+jobTriggerStrategy); } List<TriggerElement> triggerElements = new ArrayList<>(); @@ -418,21 +415,19 @@ public class TriggerConfiguration{ } //ToDo add clever Strategy to concat more than two conditions if needed - - return new TriggerConfiguration(TriggerType.S7_TRIGGER_VAR,scheduledMs,triggerElements,triggeredScrapeJob); case SCHEDULED: if(triggerVar !=null || comparatorString!=null || comparatorVariable!=null){ - throw new ScraperException("SCHEDULED trigger strategy must only be used with scheduled interval - nothing more! given configString: "+jobTriggerStrategy); + throw new ScraperConfigurationException("SCHEDULED trigger strategy must only be used with scheduled interval - nothing more! given configString: "+jobTriggerStrategy); } return new TriggerConfiguration(TriggerType.SCHEDULED,scheduledMs); default: - throw new ScraperException("Unknown Trigger Strategy "+triggerStrategy); + throw new ScraperConfigurationException("Unknown Trigger Strategy "+triggerStrategy); } } - throw new ScraperException("Invalid trigger strategy string description: "+jobTriggerStrategy); + throw new ScraperConfigurationException("Invalid trigger strategy string description: "+jobTriggerStrategy); } private void handleException(Exception e){ @@ -533,7 +528,7 @@ public class TriggerConfiguration{ this.plcField = plcField; } - TriggerElement(String comparator, String concatType, String compareValue, String plcField, String triggerStrategy) throws ScraperException { + TriggerElement(String comparator, String concatType, String compareValue, String plcField, String triggerStrategy) throws ScraperConfigurationException { this(); this.plcFieldString = plcField; this.plcConnectionString = plcConnectionString; @@ -545,7 +540,7 @@ public class TriggerConfiguration{ if(logger.isDebugEnabled()) { logger.debug("Exception occurred parsing a S7Field"); } - throw new ScraperException("Exception on parsing S7Field (" + plcField + "): " + e.getMessage()); + throw new ScraperConfigurationException("Exception on parsing S7Field (" + plcField + "): " + e.getMessage()); } this.compareValue = convertCompareValue(compareValue,this.plcField); this.comparatorType = detectComparatorType(comparator); @@ -563,7 +558,7 @@ public class TriggerConfiguration{ * @return converted object to needed data-type * @throws ScraperException when something does not match or parsing fails */ - private Object convertCompareValue(String compareValue, PlcField plcField) throws ScraperException { + private Object convertCompareValue(String compareValue, PlcField plcField) throws ScraperConfigurationException { Class<?> javaDataType = validateDataType(plcField); if(javaDataType.equals(Boolean.class)){ switch (compareValue){ @@ -575,7 +570,7 @@ public class TriggerConfiguration{ return false; default: String exceptionMessage = String.format("No valid compare Value at DataType Boolean for trigger: %s",compareValue); - throw new ScraperException(exceptionMessage); + throw new ScraperConfigurationException(exceptionMessage); } } if(javaDataType.equals(Double.class) @@ -593,18 +588,18 @@ public class TriggerConfiguration{ catch (Exception e){ logger.debug(e.getMessage(), e); String exceptionMessage = String.format("No valid compare Value at DataType Numeric for trigger: %s",compareValue); - throw new ScraperException(exceptionMessage); + throw new ScraperConfigurationException(exceptionMessage); } } String exceptionMessage = "Invalid Datatype detected ... should not happen and be catcht earlier - please report"; - throw new ScraperException(exceptionMessage); + throw new ScraperConfigurationException(exceptionMessage); } /** * converts parsed comparator from regex to ComparatorType * @throws ScraperException when no valid comparator has been used */ - private Comparator detectComparatorType(String comparator) throws ScraperException { + private Comparator detectComparatorType(String comparator) throws ScraperConfigurationException { switch (comparator){ case "==": return Comparator.EQUAL; @@ -619,7 +614,7 @@ public class TriggerConfiguration{ case ">": return Comparator.GREATER; default: - throw new ScraperException("Invalid comparator detected!"); + throw new ScraperConfigurationException("Invalid comparator detected!"); } } @@ -627,7 +622,7 @@ public class TriggerConfiguration{ * convertes parsed comparator from regex to ComparatorType * @throws ScraperException when no valid comparator has been used */ - private ConcatType detectConcatType(String concat) throws ScraperException { + private ConcatType detectConcatType(String concat) throws ScraperConfigurationException { //concat is not necessary in every case, correct usage is checked later on if(concat==null){ return null; @@ -638,7 +633,7 @@ public class TriggerConfiguration{ case "OR": return ConcatType.OR; default: - throw new ScraperException("Invalid concat between triggerVars detected: "+concat); + throw new ScraperConfigurationException("Invalid concat between triggerVars detected: "+concat); } } @@ -646,11 +641,11 @@ public class TriggerConfiguration{ * matches data-type and comparator for a valid combination * @throws ScraperException when invalid combination is detected */ - private void matchTypeAndComparator() throws ScraperException { + private void matchTypeAndComparator() throws ScraperConfigurationException { if(validateDataType(this.plcField).equals(Boolean.class) && !(this.comparatorType.equals(Comparator.EQUAL) || this.comparatorType.equals(Comparator.UNEQUAL))){ String exceptionMessage = String.format("Trigger-Data-Type (%s) and Comparator (%s) do not match",this.plcField.getDefaultJavaType(),this.comparatorType); - throw new ScraperException(exceptionMessage); + throw new ScraperConfigurationException(exceptionMessage); } //all other combinations are valid } diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/collector/TriggerCollector.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/collector/TriggerCollector.java index 37d2eed..5e1ea0f 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/collector/TriggerCollector.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/collector/TriggerCollector.java @@ -21,7 +21,8 @@ package org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector; import org.apache.plc4x.java.scraper.exception.ScraperException; /** - * defines the interface for implementing a TriggerCollector that handles and acquires all triggerRequests that need a PlcConnection + * defines the interface for implementing a TriggerCollector + * that handles and acquires all triggerRequests at once that needs a PlcConnection */ public interface TriggerCollector { diff --git a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/collector/TriggerCollectorImpl.java b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/collector/TriggerCollectorImpl.java index 02d158b..a87cc73 100644 --- a/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/collector/TriggerCollectorImpl.java +++ b/plc4j/utils/scraper/src/main/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/collector/TriggerCollectorImpl.java @@ -21,11 +21,8 @@ package org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.plc4x.java.PlcDriverManager; import org.apache.plc4x.java.api.PlcConnection; -import org.apache.plc4x.java.api.exceptions.PlcConnectionException; -import org.apache.plc4x.java.api.exceptions.PlcRuntimeException; import org.apache.plc4x.java.api.messages.PlcReadRequest; import org.apache.plc4x.java.api.messages.PlcReadResponse; -import org.apache.plc4x.java.scraper.Scraper; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl; import org.slf4j.Logger; @@ -43,32 +40,40 @@ public class TriggerCollectorImpl implements TriggerCollector { private static final Logger logger = LoggerFactory.getLogger( TriggerCollectorImpl.class ); private static final int DEFAULT_SCHEDULED_TRIGGER_INTERVAL = 1000; - private static final int FUTURE_TIMEOUT = 2000; - private static final int READ_REQUEST_TIMEOUT = 2000; + private static final int FUTURE_TIMEOUT = 2000; + private static final int READ_REQUEST_TIMEOUT = 2000; + private final PlcDriverManager plcDriverManager; private final Map<String,RequestElement> currentRequestElements; private long schedulerInterval; private final long futureTimeout; - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(3, - new BasicThreadFactory.Builder() - .namingPattern("triggercollector-scheduler-thread-%d") - .daemon(false) - .build() - ); - private final ExecutorService executorService = Executors.newFixedThreadPool(10, - new BasicThreadFactory.Builder() - .namingPattern("triggercollector-executer-thread-%d") - .daemon(true) - .build() - ); - + private final ScheduledExecutorService scheduledExecutorService; + private final ExecutorService executorService; - public TriggerCollectorImpl(PlcDriverManager plcDriverManager, long schedulerInterval, long futureTimeout) { + public TriggerCollectorImpl(PlcDriverManager plcDriverManager, long schedulerInterval, long futureTimeout, int poolSizeScheduler, int poolSizeExecutor) { this.plcDriverManager = plcDriverManager; this.currentRequestElements = new ConcurrentHashMap<>(); this.schedulerInterval = schedulerInterval; this.futureTimeout = futureTimeout; + + this.scheduledExecutorService = Executors.newScheduledThreadPool(poolSizeScheduler, + new BasicThreadFactory.Builder() + .namingPattern("triggercollector-scheduledExecutorService-thread-%d") + .daemon(false) + .build() + ); + this.executorService = Executors.newFixedThreadPool(poolSizeExecutor, + new BasicThreadFactory.Builder() + .namingPattern("triggercollector-executerService-thread-%d") + .daemon(true) + .build() + ); + + } + + public TriggerCollectorImpl(PlcDriverManager plcDriverManager, long schedulerInterval, long futureTimeout) { + this(plcDriverManager,schedulerInterval,futureTimeout,10,20); } public TriggerCollectorImpl(PlcDriverManager plcDriverManager) { @@ -212,14 +217,8 @@ public class TriggerCollectorImpl implements TriggerCollector { * @return the object acquired by requesting plc instance */ @Override - public Object requestResult(String uuid, long timeout) throws ScraperException { - Object result = currentRequestElements.get(uuid).getResult(); - /* - if(result==null){ - throw new ScraperException("No result acquired yet"); - } - */ - return result; + public Object requestResult(String uuid, long timeout){ + return currentRequestElements.get(uuid).getResult(); } /** @@ -227,7 +226,7 @@ public class TriggerCollectorImpl implements TriggerCollector { */ @Override public void start() { - this.scheduler.scheduleAtFixedRate(() -> processActiveTrigger(), 1_000, this.schedulerInterval, TimeUnit.MILLISECONDS); + this.scheduledExecutorService.scheduleAtFixedRate(this::processActiveTrigger, 1_000, this.schedulerInterval, TimeUnit.MILLISECONDS); } /** @@ -235,52 +234,37 @@ public class TriggerCollectorImpl implements TriggerCollector { */ @Override public void stop() { - //ToDo stop everything the right way + this.scheduledExecutorService.shutdown(); + this.executorService.shutdown(); } + class RequestElement{ private String plcConnectionString; private String plcField; - private LocalDateTime submitTimeOut; private LocalDateTime lastAcquirement; private Object result; private String uuid; - private boolean submitted; - private CompletableFuture<PlcReadResponse> plcReadResponse; private long scanIntervalMs; - public RequestElement(String plcConnectionString, String plcField, long scanIntervalMs, String uuid) { + RequestElement(String plcConnectionString, String plcField, long scanIntervalMs, String uuid) { this.plcConnectionString = plcConnectionString; this.plcField = plcField; - this.submitted = false; - this.submitTimeOut = LocalDateTime.now().plus(scanIntervalMs, ChronoUnit.MILLIS); this.uuid = uuid; this.scanIntervalMs = scanIntervalMs; //set initial acquirement to a long time ago this.lastAcquirement = LocalDateTime.of(1,1,1,1,1,1); } - public String getPlcConnectionString() { + String getPlcConnectionString() { return plcConnectionString; } - public String getPlcField() { + String getPlcField() { return plcField; } - public LocalDateTime getSubmitTimeOut() { - return submitTimeOut; - } - - public boolean isSubmitted() { - return submitted; - } - - public void setSubmitted(boolean submitted) { - this.submitted = submitted; - } - public Object getResult() { return result; } @@ -289,31 +273,23 @@ public class TriggerCollectorImpl implements TriggerCollector { this.result = result; } - public String getUuid() { + String getUuid() { return uuid; } - public CompletableFuture<PlcReadResponse> getPlcReadResponse() { - return plcReadResponse; - } - - public void setPlcReadResponse(CompletableFuture<PlcReadResponse> plcReadResponse) { - this.plcReadResponse = plcReadResponse; - } - - public long getScanIntervalMs() { + long getScanIntervalMs() { return scanIntervalMs; } - public void setScanIntervalMs(long scanIntervalMs) { + void setScanIntervalMs(long scanIntervalMs) { this.scanIntervalMs = scanIntervalMs; } - public LocalDateTime getLastAcquirement() { + LocalDateTime getLastAcquirement() { return lastAcquirement; } - public void setLastAcquirement(LocalDateTime lastAcquirement) { + void setLastAcquirement(LocalDateTime lastAcquirement) { this.lastAcquirement = lastAcquirement; } @@ -330,6 +306,18 @@ public class TriggerCollectorImpl implements TriggerCollector { public int hashCode() { return Objects.hash(plcConnectionString, plcField); } + + @Override + public String toString() { + return "RequestElement{" + + "plcConnectionString='" + plcConnectionString + '\'' + + ", plcField='" + plcField + '\'' + + ", lastAcquirement=" + lastAcquirement + + ", result=" + result + + ", uuid='" + uuid + '\'' + + ", scanIntervalMs=" + scanIntervalMs + + '}'; + } } } diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java index a153b1c..11f4c35 100644 --- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java +++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperConfigurationTest.java @@ -24,8 +24,9 @@ import com.fasterxml.jackson.databind.exc.MismatchedInputException; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.apache.plc4x.java.api.exceptions.PlcRuntimeException; import org.apache.plc4x.java.scraper.config.JobConfiguration; -import org.apache.plc4x.java.scraper.config.JobConfigurationImpl; +import org.apache.plc4x.java.scraper.config.JobConfigurationClassicImpl; import org.apache.plc4x.java.scraper.config.ScraperConfiguration; +import org.apache.plc4x.java.scraper.config.ScraperConfigurationClassicImpl; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.assertj.core.api.WithAssertions; import org.junit.jupiter.api.Nested; @@ -58,7 +59,7 @@ class ScraperConfigurationTest implements WithAssertions { " a: DBasdf\n" + " b: DBbsdf\n"; - ScraperConfiguration configuration = mapper.readValue(yaml, ScraperConfiguration.class); + ScraperConfiguration configuration = mapper.readValue(yaml, ScraperConfigurationClassicImpl.class); assertThat(configuration.getJobConfigurations()).hasSize(1); JobConfiguration conf = configuration.getJobConfigurations().get(0); @@ -71,7 +72,7 @@ class ScraperConfigurationTest implements WithAssertions { .containsEntry("a3", "b"); assertThat(conf.getName()).isEqualTo("job1"); - assertThat(((JobConfigurationImpl)conf).getScrapeRate()).isEqualTo(10); + assertThat(((JobConfigurationClassicImpl)conf).getScrapeRate()).isEqualTo(10); assertThat(conf.getSources()) .hasSize(3); @@ -91,7 +92,7 @@ class ScraperConfigurationTest implements WithAssertions { " sources:\n" + " - a1\n"; - assertThatThrownBy(() -> mapper.readValue(jobs, ScraperConfiguration.class)) + assertThatThrownBy(() -> mapper.readValue(jobs, ScraperConfigurationClassicImpl.class)) .isInstanceOf(MismatchedInputException.class); } @@ -112,7 +113,7 @@ class ScraperConfigurationTest implements WithAssertions { " a: DBasdf\n" + " b: DBbsdf\n"; - assertThatCode(() -> ScraperConfiguration.fromYaml(yaml)) + assertThatCode(() -> ScraperConfiguration.fromYaml(yaml, ScraperConfigurationClassicImpl.class)) .doesNotThrowAnyException(); } @@ -141,7 +142,7 @@ class ScraperConfigurationTest implements WithAssertions { " ]\n" + "}"; - assertThatCode(() -> ScraperConfiguration.fromJson(json)) + assertThatCode(() -> ScraperConfiguration.fromJson(json, ScraperConfigurationClassicImpl.class)) .doesNotThrowAnyException(); } @@ -156,7 +157,7 @@ class ScraperConfigurationTest implements WithAssertions { " - s1\n" + " fields:\n"; - assertThatThrownBy(() -> ScraperConfiguration.fromYaml(yaml)) + assertThatThrownBy(() -> ScraperConfiguration.fromYaml(yaml, ScraperConfigurationClassicImpl.class)) .isInstanceOf(PlcRuntimeException.class) .hasStackTraceContaining("unreferenced sources: [s1]"); } @@ -173,7 +174,7 @@ class ScraperConfigurationTest implements WithAssertions { " fields:\n" + " field1: 'DB1 Field 1'\n"; - List<ScrapeJob> jobs = ScraperConfiguration.fromYaml(yaml).getJobs(); + List<ScrapeJob> jobs = ScraperConfiguration.fromYaml(yaml, ScraperConfigurationClassicImpl.class).getJobs(); assertThat(jobs).hasSize(1); ScrapeJob job = jobs.get(0); @@ -193,12 +194,12 @@ class ScraperConfigurationTest implements WithAssertions { @Test void json() throws IOException { - ScraperConfiguration conf = ScraperConfiguration.fromFile("src/test/resources/config.json"); + ScraperConfiguration conf = ScraperConfiguration.fromFile("src/test/resources/config.json", ScraperConfigurationClassicImpl.class); } @Test void yaml() throws IOException { - ScraperConfiguration conf = ScraperConfiguration.fromFile("src/test/resources/config.yml"); + ScraperConfiguration conf = ScraperConfiguration.fromFile("src/test/resources/config.yml", ScraperConfigurationClassicImpl.class); } } } \ No newline at end of file diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java index f3548bc..3a6f543 100644 --- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java +++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/ScraperRunner.java @@ -20,6 +20,7 @@ package org.apache.plc4x.java.scraper; import org.apache.plc4x.java.scraper.config.ScraperConfiguration; +import org.apache.plc4x.java.scraper.config.ScraperConfigurationClassicImpl; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +32,7 @@ public class ScraperRunner { private static final Logger LOGGER = LoggerFactory.getLogger(ScraperRunner.class); public static void main(String[] args) throws IOException, ScraperException { - ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example.yml"); + ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example.yml", ScraperConfigurationClassicImpl.class); Scraper scraper = new ScraperImpl(configuration, (j, a, m) -> LOGGER.info("Results from {}/{}: {}", j, a, m)); scraper.start(); diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/TriggeredScraperRunner.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/TriggeredScraperRunner.java index 5c22893..545b4aa 100644 --- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/TriggeredScraperRunner.java +++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/TriggeredScraperRunner.java @@ -20,7 +20,8 @@ package org.apache.plc4x.java.scraper; import org.apache.plc4x.java.PlcDriverManager; -import org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredScraperConfiguration; +import org.apache.plc4x.java.scraper.config.ScraperConfiguration; +import org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImpl; import org.apache.plc4x.java.scraper.exception.ScraperException; @@ -41,7 +42,7 @@ public class TriggeredScraperRunner { * testing of TriggeredScraper vs real device */ public static void main(String[] args) throws IOException, ScraperException { - TriggeredScraperConfiguration configuration = TriggeredScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example_triggered_scraper.yml"); + ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example_triggered_scraper.yml", ScraperConfigurationTriggeredImpl.class); PlcDriverManager plcDriverManager = new PooledPlcDriverManager(); TriggerCollector triggerCollector = new TriggerCollectorImpl(plcDriverManager); diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/TriggeredScraperRunnerModbus.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/TriggeredScraperRunnerModbus.java index 345b3b8..fd3fe3a 100644 --- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/TriggeredScraperRunnerModbus.java +++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/TriggeredScraperRunnerModbus.java @@ -21,7 +21,8 @@ package org.apache.plc4x.java.scraper; import org.apache.plc4x.java.PlcDriverManager; import org.apache.plc4x.java.modbus.connection.ModbusConnectionFactory; -import org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredScraperConfiguration; +import org.apache.plc4x.java.scraper.config.ScraperConfiguration; +import org.apache.plc4x.java.scraper.config.triggeredscraper.ScraperConfigurationTriggeredImpl; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScraperImpl; import org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector.TriggerCollectorImpl; @@ -41,7 +42,7 @@ public class TriggeredScraperRunnerModbus { * testing of TriggeredScraper vs real device (Modbus) */ public static void main(String[] args) throws IOException, ScraperException { - TriggeredScraperConfiguration configuration = TriggeredScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example_triggered_scraper_modbus.yml"); + ScraperConfiguration configuration = ScraperConfiguration.fromFile("plc4j/utils/scraper/src/test/resources/example_triggered_scraper_modbus.yml", ScraperConfigurationTriggeredImpl.class); PlcDriverManager plcDriverManager = new PooledPlcDriverManager(); TriggeredScraperImpl scraper = new TriggeredScraperImpl( configuration, diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationBuilderTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationBuilderTest.java index 71b16a8..5ce58eb 100644 --- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationBuilderTest.java +++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/config/ScraperConfigurationBuilderTest.java @@ -23,8 +23,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import org.assertj.core.api.WithAssertions; -import org.junit.Ignore; -import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.List; @@ -33,18 +31,18 @@ class ScraperConfigurationBuilderTest implements WithAssertions { //ToDo test is failing idon't know why (Tim) void builder_usage_example() throws JsonProcessingException { - ScraperConfigurationBuilder builder = new ScraperConfigurationBuilder(); + ScraperConfigurationClassicImplBuilder builder = new ScraperConfigurationClassicImplBuilder(); List<String> sources = Arrays.asList("s1", "s2"); List<String> jobs = Arrays.asList("j1", "j2"); sources.forEach(source -> builder.addSource(source, source)); for (String job : jobs) { - JobConfigurationImplBuilder jobConfigurationImplBuilder = builder.job(job, 10); - sources.forEach(jobConfigurationImplBuilder::source); + JobConfigurationClassicImplBuilder jobConfigurationClassicImplBuilder = builder.job(job, 10); + sources.forEach(jobConfigurationClassicImplBuilder::source); for (int i = 1; i <= 10; i++) { - jobConfigurationImplBuilder.field("f" + i, "qry" + i); + jobConfigurationClassicImplBuilder.field("f" + i, "qry" + i); } - jobConfigurationImplBuilder.build(); + jobConfigurationClassicImplBuilder.build(); } ScraperConfiguration configuration = builder.build(); diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperImplTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperImplTest.java index c47b938..d1bbefa 100644 --- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperImplTest.java +++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/triggeredscraper/TriggeredScraperImplTest.java @@ -21,13 +21,13 @@ package org.apache.plc4x.java.scraper.triggeredscraper; import org.apache.commons.lang3.tuple.Pair; import org.apache.plc4x.java.PlcDriverManager; -import org.apache.plc4x.java.api.PlcConnection; import org.apache.plc4x.java.api.types.PlcResponseCode; import org.apache.plc4x.java.base.messages.items.DefaultBooleanFieldItem; import org.apache.plc4x.java.base.messages.items.DefaultLongFieldItem; import org.apache.plc4x.java.mock.MockDevice; import org.apache.plc4x.java.mock.PlcMockConnection; -import org.apache.plc4x.java.scraper.config.triggeredscraper.TriggeredScraperConfiguration; +import org.apache.plc4x.java.scraper.config.ScraperConfiguration; +import org.apache.plc4x.java.scraper.config.ScraperConfigurationClassicImpl; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector.TriggerCollector; import org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler.collector.TriggerCollectorImpl; @@ -42,7 +42,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Random; -import static org.junit.Assert.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; @@ -106,7 +105,7 @@ public class TriggeredScraperImplTest { when(mockDevice1.read(eq("%DB810:DBW0:INT"))).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultLongFieldItem(3L))); when(mockDevice2.read(eq("%DB810:DBW0:INT"))).thenReturn(Pair.of(PlcResponseCode.OK, new DefaultLongFieldItem(4L))); - TriggeredScraperConfiguration configuration = TriggeredScraperConfiguration.fromFile("src/test/resources/mock-scraper-config.yml"); + ScraperConfiguration configuration = ScraperConfiguration.fromFile("src/test/resources/mock-scraper-config.yml", ScraperConfigurationClassicImpl.class); TriggerCollector triggerCollector = new TriggerCollectorImpl(driverManager); TriggeredScraperImpl scraper = new TriggeredScraperImpl((j, a, m) -> System.out.println(String.format("Results from %s/%s: %s", j, a, m)), driverManager, configuration.getJobs(),triggerCollector,1000); diff --git a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/TriggerConfigurationTest.java b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/TriggerConfigurationTest.java index 68402a9..f2c1810 100644 --- a/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/TriggerConfigurationTest.java +++ b/plc4j/utils/scraper/src/test/java/org/apache/plc4x/java/scraper/triggeredscraper/triggerhandler/TriggerConfigurationTest.java @@ -19,6 +19,7 @@ package org.apache.plc4x.java.scraper.triggeredscraper.triggerhandler; +import org.apache.plc4x.java.scraper.exception.ScraperConfigurationException; import org.apache.plc4x.java.scraper.exception.ScraperException; import org.apache.plc4x.java.scraper.triggeredscraper.TriggeredScrapeJobImpl; import org.apache.plc4x.test.FastTests; @@ -106,7 +107,7 @@ class TriggerConfigurationTest { TriggerConfiguration triggerConfiguration = null; try { triggerConfiguration = TriggerConfiguration.createConfiguration(triggerConfig,triggeredScrapeJob); - } catch (ScraperException e) { + } catch (ScraperConfigurationException e) { //should not happen } @@ -139,7 +140,7 @@ class TriggerConfigurationTest { triggerConfiguration = TriggerConfiguration.createConfiguration(triggerConfig,triggeredScrapeJob); assertThat(triggerConfiguration,null); //NPE should happen when test fails! - } catch (ScraperException e) { + } catch (ScraperConfigurationException e) { LOGGER.info("Exception as expected for positive test result: {}",e.getMessage()); //should happen }
