This is an automated email from the ASF dual-hosted git repository.
riemer pushed a change to branch minor-layout-harmonization
in repository https://gitbox.apache.org/repos/asf/streampipes.git
from 55c11cd481 Fix styling, add cypress tag
add 8893b6852d Small fix for the failing test in PipelineAsset. (#3884)
add dce7d250ab fix(3887): permission objects lost after exporting and re
importing assets with sanitized resource ids (#3888)
add e776239bcc refactor: Use nats as default and clean up of readme (#3886)
add 93d9179782 [maven-release-plugin] prepare branch @{releaseLabel}
add 2a122df84f [maven-release-plugin] prepare for next development
iteration
add ecaa286dbf feat: Add loadbalancer (#3795)
add 7ce00b2233 feat:Add SpRateLimiter and SpMemoryManager (#3796)
add e940e092b0 fix: Access rights for adapter and pipeline endpoints
(#3890)
add 2661dd08f1 fix: Avoid sending events twice in adapter sink (#3892)
add 85c51cde18 fix(#3893): Fix translation bug in data lake configuration
(#3894)
add d866764436 feat: Bump PLC4X version, improve connection cache
management (#3744)
add 23257ffba9 fix(#3893): Fix handle failing plc read (#3899)
add 152862830f fix: Mark old mqtt based adapters as deprecated for future
removal (#3900)
add beae74fbb7 fix: Only admins can see dashboard permissions dialog
(#3901)
add 9c862aaf11 fix: Bump remaining snapshot versions to 0.99 (#3902)
add f4d95e5932 Merge branch 'dev' into minor-layout-harmonization
No new revisions were added by this update.
Summary of changes:
.../streampipes-archetype-extensions-jvm/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
.../pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
.../streampipes-archetype-pe-sinks-flink/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
installer/cli/.env | 2 +-
installer/cli/README.md | 2 +-
installer/compose/.env | 2 +-
installer/compose/README.md | 50 +-
...r-compose.full.yml => docker-compose.kafka.yml} | 39 +-
installer/compose/docker-compose.nats.yml | 114 ----
installer/compose/docker-compose.pulsar.yml | 109 ----
installer/compose/docker-compose.yml | 45 +-
installer/k8s/Chart.yaml | 2 +-
installer/k8s/README.md | 4 +-
installer/k8s/values.yaml | 2 +-
pom.xml | 16 +-
streampipes-client-api/pom.xml | 8 +-
streampipes-client/pom.xml | 14 +-
streampipes-commons/pom.xml | 2 +-
.../apache/streampipes/commons/constants/Envs.java | 71 ++-
.../commons/environment/DefaultEnvironment.java | 172 +++++-
.../commons/environment/Environment.java | 95 +++-
.../commons/environment/Environments.java | 1 -
.../variable/BooleanEnvironmentVariable.java | 1 -
...ariable.java => DoubleEnvironmentVariable.java} | 17 +-
...Variable.java => FloatEnvironmentVariable.java} | 17 +-
.../variable/IntEnvironmentVariable.java | 1 -
...tVariable.java => LongEnvironmentVariable.java} | 10 +-
.../variable/StringEnvironmentVariable.java | 1 -
.../commons/exceptions/SpHttpErrorStatusCode.java | 2 +-
.../prometheus/StreamPipesCollectorRegistry.java | 47 +-
.../loadbalancer/LoadBalancerMetrics.java | 50 ++
.../prometheus/loadbalancer/LoadBalancerStats.java | 52 ++
.../prometheus/service/ElementServiceMetrics.java | 58 +--
.../prometheus/service/ElementServiceStats.java | 3 -
.../spmemorymanager/SpMemoryManagerMetrics.java | 30 +-
.../spmemorymanager/SpMemoryManagerStats.java | 66 +++
.../SpRateLimiterMetrics.java} | 37 +-
.../spratelimiter/SpRateLimiterStats.java | 66 +++
streampipes-connect-management/pom.xml | 12 +-
.../management/health/AdapterHealthCheck.java | 167 +++---
.../management/AdapterMasterManagement.java | 172 +++---
streampipes-connect-shared/pom.xml | 10 +-
streampipes-data-explorer-api/pom.xml | 8 +-
streampipes-data-explorer-export/pom.xml | 8 +-
streampipes-data-explorer-influx/pom.xml | 16 +-
streampipes-data-explorer-iotdb/pom.xml | 16 +-
streampipes-data-explorer-management/pom.xml | 10 +-
streampipes-data-explorer/pom.xml | 10 +-
streampipes-data-export/pom.xml | 12 +-
.../export/dataimport/PerformImportGenerator.java | 12 +-
streampipes-dataformat/pom.xml | 4 +-
streampipes-extensions-api/pom.xml | 33 +-
.../extensions/api/limiter/SpRateLimiter.java | 307 +++++++++++
.../api/memorymanager/SpMemoryManager.java | 331 ++++++++++++
.../api/monitoring/SpMonitoringManager.java | 6 +-
.../api/pe/routing/RawDataProcessor.java | 2 +-
streampipes-extensions-management/pom.xml | 22 +-
.../connect/HttpServerAdapterManagement.java | 2 +-
.../management/connect/PullAdapterScheduler.java | 5 +-
.../elements/SendToBrokerAdapterSink.java | 8 +-
.../monitoring/ServiceLoadDataReportGenerator.java | 102 ++--
streampipes-extensions/pom.xml | 4 +-
.../streampipes-connect-adapters-iiot/pom.xml | 10 +-
.../iiot/adapters/iolink/IfmAlMqttAdapter.java | 1 +
.../streampipes-connect-adapters/pom.xml | 10 +-
.../connect/adapters/netio/NetioMQTTAdapter.java | 1 +
.../connect/adapters/netio/NetioRestAdapter.java | 1 +
.../connect/adapters/netio/NetioUtils.java | 1 +
.../streampipes-connectors-influx/pom.xml | 8 +-
.../streampipes-connectors-kafka/pom.xml | 14 +-
.../streampipes-connectors-mqtt/pom.xml | 8 +-
.../streampipes-connectors-nats/pom.xml | 10 +-
.../streampipes-connectors-opcua/pom.xml | 10 +-
.../streampipes-connectors-plc/pom.xml | 14 +-
.../connectors/plc/PlcConnectorsModuleExport.java | 4 +-
.../connection/ContinuousPlcRequestReader.java | 38 +-
.../generic/connection/PlcEventGenerator.java | 7 +-
.../generic/model/Plc4xConnectionExtractor.java | 11 +-
.../connectors/plc/adapter/s7/Plc4xS7Adapter.java | 2 +-
.../plc/cache/SpCachedPlcConnectionManager.java | 178 +++++++
.../plc/cache/SpConnectionContainer.java | 235 +++++++++
.../plc/cache/SpLeasedPlcConnection.java | 575 +++++++++++++++++++++
.../plc/adapter/ConnectionContainerReproTest.java | 167 ++++++
.../streampipes-connectors-pulsar/pom.xml | 8 +-
.../streampipes-connectors-rocketmq/pom.xml | 8 +-
.../rocketmq/adapter/RocketMQConsumer.java | 10 +-
.../rocketmq/sink/RocketMQPublisherSink.java | 2 +-
.../streampipes-connectors-ros/pom.xml | 10 +-
.../streampipes-connectors-tubemq/pom.xml | 8 +-
.../streampipes-extensions-all-iiot/pom.xml | 50 +-
.../streampipes-extensions-all-jvm/pom.xml | 62 +--
.../streampipes-extensions-iiot-minimal/pom.xml | 40 +-
.../pom.xml | 6 +-
.../streampipes-pipeline-elements-shared/pom.xml | 4 +-
.../pom.xml | 6 +-
.../streampipes-processors-enricher-jvm/pom.xml | 10 +-
.../streampipes-processors-filters-jvm/pom.xml | 8 +-
.../streampipes-processors-filters-siddhi/pom.xml | 8 +-
.../streampipes-processors-geo-jvm/pom.xml | 6 +-
.../pom.xml | 6 +-
.../streampipes-processors-llm-jvm/pom.xml | 8 +-
.../streampipes-processors-text-mining-jvm/pom.xml | 6 +-
.../pom.xml | 10 +-
.../streampipes-sinks-brokers-jvm/pom.xml | 10 +-
.../streampipes-sinks-databases-jvm/pom.xml | 6 +-
.../streampipes-sinks-internal-jvm/pom.xml | 12 +-
.../streampipes-sinks-notifications-jvm/pom.xml | 8 +-
streampipes-integration-tests/pom.xml | 22 +-
.../pom.xml | 48 +-
.../loadbalance/ExtensionServiceSelector.java | 61 +++
.../streampipes/loadbalance/LoadBalancer.java | 26 +-
.../loadbalance/LoadBalancerConfig.java | 75 +++
.../streampipes/loadbalance/LoadManager.java | 240 +++++++++
.../streampipes/loadbalance/PipelineMigrator.java | 18 +-
.../loadbalance/ResourceUnitMigration.java | 330 ++++++++++++
.../loadbalance/ServiceLoadCalculator.java | 135 +++++
.../loadbalance/impl/AbstractPipelineMigrator.java | 208 ++++++++
.../loadbalance/impl/ExtensibleLoadManager.java | 60 +++
.../loadbalance/impl/MinimumLoadSelector.java | 124 +++++
.../loadbalance/impl/OverloadMigrator.java | 93 ++++
.../loadbalance/impl/ThresholdMigrator.java | 138 +++++
.../loadbalance/impl/TransferMigrator.java | 145 ++++++
.../loadbalance/impl/WeightedFirstSelector.java | 172 ++++++
.../loadbalance/impl/WeightedRandomSelector.java | 155 ++++++
.../pipeline/ExtensionsLogProvider.java | 25 +-
.../pipeline/ExtensionsServiceLogExecutor.java | 60 +--
.../service}/ExtensionServiceExecutions.java | 33 +-
.../service/ExtensionsServiceReportExecutor.java | 89 ++++
.../unit}/ExtensionsServiceEndpointUtils.java | 2 +-
.../loadbalance/unit/InvokeHttpRequest.java | 131 +++++
.../unit/PipelineElementPartitioner.java | 458 ++++++++++++++++
.../loadbalance/unit/ResourceUnitScanner.java | 370 +++++++++++++
.../loadbalance/unit/ResourceUnitStatsScanner.java | 345 +++++++++++++
streampipes-mail/pom.xml | 4 +-
streampipes-maven-plugin/pom.xml | 4 +-
streampipes-measurement-units/pom.xml | 4 +-
streampipes-messaging-jms/pom.xml | 4 +-
.../messaging/jms/ActiveMQConsumer.java | 10 +-
streampipes-messaging-kafka/pom.xml | 6 +-
.../messaging/kafka/SpKafkaConsumer.java | 9 +-
streampipes-messaging-mqtt/pom.xml | 4 +-
streampipes-messaging-nats/pom.xml | 4 +-
streampipes-messaging-pulsar/pom.xml | 4 +-
.../messaging/pulsar/PulsarConsumer.java | 10 +-
streampipes-messaging/pom.xml | 4 +-
.../messaging/InternalEventProcessor.java | 2 +-
streampipes-model-client/pom.xml | 6 +-
streampipes-model-shared/pom.xml | 2 +-
streampipes-model/pom.xml | 8 +-
.../svcdiscovery/SpServiceRegistration.java | 17 +
.../svcdiscovery/SpServiceTagPrefix.java | 5 +
.../loadbalancer/LoadBalanceResourceUnit.java | 120 +++++
.../loadbalancer/LoadBalanceResourceUnitStats.java | 154 ++++++
.../model/loadbalancer/ServiceLoadDataReport.java | 249 +++++++--
.../model/monitoring/MessageCounter.java | 9 +
.../model/monitoring/SpMetricsEntry.java | 5 +-
.../streampipes/model/pipeline/Pipeline.java | 20 +
streampipes-pipeline-management/pom.xml | 36 +-
.../ExtensionsServiceEndpointGenerator.java | 100 ++--
.../execution/task/DiscoverEndpointsTask.java | 32 +-
.../manager/health/PipelineHealthCheck.java | 81 ++-
.../manager/health/ServiceHealthCheck.java | 55 +-
.../manager/health/ServiceRegistrationManager.java | 28 +-
.../manager/pipeline/PipelineManager.java | 6 +-
streampipes-resource-management/pom.xml | 8 +-
streampipes-rest-core-base/pom.xml | 10 +-
streampipes-rest-extensions/pom.xml | 8 +-
streampipes-rest-shared/pom.xml | 6 +-
streampipes-rest/pom.xml | 31 +-
.../rest/impl/AbstractMonitoringResource.java | 2 +-
.../rest/impl/AdapterMonitoringResource.java | 66 ++-
.../streampipes/rest/impl/FunctionsResource.java | 2 +-
.../streampipes/rest/impl/PipelineMonitoring.java | 8 +-
.../streampipes/rest/impl/PipelineResource.java | 10 +-
.../impl/admin/ServiceRegistrationResource.java | 8 +-
.../rest/impl/connect/AdapterResource.java | 187 ++++---
.../rest/impl/connect/CompactAdapterResource.java | 2 +-
.../rest/impl/datalake/DataLakeWidgetResource.java | 2 +-
streampipes-sdk-bundle/pom.xml | 18 +-
streampipes-sdk/pom.xml | 10 +-
streampipes-security-jwt/pom.xml | 4 +-
streampipes-serializers-json/pom.xml | 8 +-
streampipes-service-base/pom.xml | 6 +-
streampipes-service-core-minimal/pom.xml | 6 +-
streampipes-service-core/pom.xml | 28 +-
.../service/core/StreamPipesCoreApplication.java | 117 ++---
.../core/migrations/AvailableMigrations.java | 4 +-
.../v0980/FixImportedPermissionsMigration.java | 116 +++++
streampipes-service-discovery-api/pom.xml | 4 +-
.../svcdiscovery/api/ISpServiceDiscovery.java | 9 +
streampipes-service-discovery/pom.xml | 8 +-
.../svcdiscovery/SpServiceDiscoveryCore.java | 16 +
streampipes-service-extensions/pom.xml | 6 +-
.../StreamPipesExtensionsServiceBase.java | 2 +
streampipes-storage-api/pom.xml | 8 +-
.../streampipes/storage/api/IPipelineStorage.java | 1 -
streampipes-storage-couchdb/pom.xml | 10 +-
.../storage/couchdb/impl/PipelineStorageImpl.java | 12 +-
streampipes-storage-management/pom.xml | 6 +-
streampipes-test-utils-executors/pom.xml | 6 +-
streampipes-test-utils/pom.xml | 8 +-
streampipes-user-management/pom.xml | 8 +-
streampipes-vocabulary/pom.xml | 2 +-
streampipes-wrapper-distributed/pom.xml | 6 +-
streampipes-wrapper-flink/pom.xml | 8 +-
streampipes-wrapper-kafka-streams/pom.xml | 6 +-
streampipes-wrapper-siddhi/pom.xml | 4 +-
streampipes-wrapper-standalone/pom.xml | 6 +-
.../standalone/function/StreamPipesFunction.java | 9 +-
.../routing/StandaloneSpInputCollector.java | 9 +-
.../routing/StandaloneSpOutputCollector.java | 5 +-
.../runtime/StandaloneEventProcessorRuntime.java | 13 +-
.../runtime/StandaloneEventSinkRuntime.java | 13 +-
streampipes-wrapper/pom.xml | 14 +-
ui/cypress/tests/pipeline/pipelineAsset.spec.ts | 2 +-
ui/deployment/i18n/de.json | 2 +
ui/deployment/i18n/en.json | 2 +
ui/package-lock.json | 4 +-
ui/package.json | 2 +-
.../dashboard-overview-table.component.html | 1 +
223 files changed, 7886 insertions(+), 1674 deletions(-)
rename installer/compose/{docker-compose.full.yml => docker-compose.kafka.yml}
(83%)
delete mode 100644 installer/compose/docker-compose.nats.yml
delete mode 100644 installer/compose/docker-compose.pulsar.yml
copy
streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/variable/{StringEnvironmentVariable.java
=> DoubleEnvironmentVariable.java} (71%)
copy
streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/variable/{StringEnvironmentVariable.java
=> FloatEnvironmentVariable.java} (71%)
copy
streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/variable/{StringEnvironmentVariable.java
=> LongEnvironmentVariable.java} (79%)
create mode 100644
streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/loadbalancer/LoadBalancerMetrics.java
create mode 100644
streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/loadbalancer/LoadBalancerStats.java
copy
streampipes-service-base/src/main/java/org/apache/streampipes/service/base/StreamPipesPrometheusConfig.java
=>
streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/spmemorymanager/SpMemoryManagerMetrics.java
(54%)
create mode 100644
streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/spmemorymanager/SpMemoryManagerStats.java
copy
streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/{pipelines/PipelineFlowMetrics.java
=> spratelimiter/SpRateLimiterMetrics.java} (54%)
create mode 100644
streampipes-commons/src/main/java/org/apache/streampipes/commons/prometheus/spratelimiter/SpRateLimiterStats.java
create mode 100644
streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/limiter/SpRateLimiter.java
create mode 100644
streampipes-extensions-api/src/main/java/org/apache/streampipes/extensions/api/memorymanager/SpMemoryManager.java
create mode 100644
streampipes-extensions/streampipes-connectors-plc/src/main/java/org/apache/streampipes/extensions/connectors/plc/cache/SpCachedPlcConnectionManager.java
create mode 100644
streampipes-extensions/streampipes-connectors-plc/src/main/java/org/apache/streampipes/extensions/connectors/plc/cache/SpConnectionContainer.java
create mode 100644
streampipes-extensions/streampipes-connectors-plc/src/main/java/org/apache/streampipes/extensions/connectors/plc/cache/SpLeasedPlcConnection.java
create mode 100644
streampipes-extensions/streampipes-connectors-plc/src/test/java/org/apache/streampipes/extensions/connectors/plc/adapter/ConnectionContainerReproTest.java
copy {streampipes-sdk-bundle => streampipes-load-balancer}/pom.xml (68%)
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/ExtensionServiceSelector.java
copy
streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/IMigrationHandler.java
=>
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/LoadBalancer.java
(59%)
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/LoadBalancerConfig.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/LoadManager.java
copy
streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/migration/IMigrationHandler.java
=>
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/PipelineMigrator.java
(66%)
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/ResourceUnitMigration.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/ServiceLoadCalculator.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/AbstractPipelineMigrator.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/ExtensibleLoadManager.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/MinimumLoadSelector.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/OverloadMigrator.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/ThresholdMigrator.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/TransferMigrator.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/WeightedFirstSelector.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/impl/WeightedRandomSelector.java
rename
{streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/monitoring
=>
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance}/pipeline/ExtensionsLogProvider.java
(85%)
rename
{streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/monitoring
=>
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance}/pipeline/ExtensionsServiceLogExecutor.java
(68%)
copy
{streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution
=>
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/service}/ExtensionServiceExecutions.java
(59%)
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/service/ExtensionsServiceReportExecutor.java
copy
{streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/endpoint
=>
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit}/ExtensionsServiceEndpointUtils.java
(97%)
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/InvokeHttpRequest.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/PipelineElementPartitioner.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitScanner.java
create mode 100644
streampipes-load-balancer/src/main/java/org/apache/streampipes/loadbalance/unit/ResourceUnitStatsScanner.java
create mode 100644
streampipes-model/src/main/java/org/apache/streampipes/model/loadbalancer/LoadBalanceResourceUnit.java
create mode 100644
streampipes-model/src/main/java/org/apache/streampipes/model/loadbalancer/LoadBalanceResourceUnitStats.java
create mode 100644
streampipes-service-core/src/main/java/org/apache/streampipes/service/core/migrations/v0980/FixImportedPermissionsMigration.java
