This is an automated email from the ASF dual-hosted git repository. riemer pushed a commit to branch 1207-improve-helm-charts in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 571f230ce4f562fc92ce6bc1463468a77a349fd9 Merge: c6f79e2eb d70630c2d Author: Dominik Riemer <[email protected]> AuthorDate: Thu Feb 23 11:25:04 2023 +0100 Merge branch 'dev' into 1207-improve-helm-charts .asf.yaml | 10 +- .github/dependabot.yml | 3 + .github/pr-labeler-config.yml | 3 + .github/workflows/build.yml | 6 +- .github/workflows/cypress-test.yml | 4 +- .github/workflows/pr-labeler.yml | 14 +- .github/workflows/pr-validation.yml | 20 +- .../{pr-labeler.yml => pypi-deployment.yml} | 47 +- .github/workflows/python-docs.yml | 59 +++ README.md | 3 +- .../streampipes-archetype-extensions-jvm/pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 2 +- .../pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 2 +- .../streampipes-archetype-pe-sinks-flink/pom.xml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 2 +- installer/cli/.env | 6 +- installer/cli/README.md | 2 +- .../deploy/standalone/epsg/docker-compose.dev.yml | 8 +- .../cli/deploy/standalone/epsg/docker-compose.yml | 44 ++ installer/cli/environments/backend | 3 +- installer/compose/.env | 2 +- installer/compose/README.md | 2 +- installer/k8s/Chart.yaml | 4 +- installer/k8s/README.md | 2 +- installer/k8s/values.yaml | 2 +- installer/scripts/epsg/EPSG_FINISH.sql | 17 + installer/scripts/epsg/PostgreSQL_Data_Script.sql | 17 + installer/scripts/epsg/PostgreSQL_FKey_Script.sql | 17 + installer/scripts/epsg/PostgreSQL_Table_Script.sql | 17 + installer/scripts/epsg/Readme.md | 140 +++++ pom.xml | 61 +-- streampipes-client-python/.pre-commit-config.yaml | 2 +- streampipes-client-python/Makefile | 14 +- streampipes-client-python/README.md | 36 +- ...introduction-to-streampipes-python-client.ipynb | 8 +- ...cting-data-from-the-streampipes-data-lake.ipynb | 12 +- ...ive-data-from-the-streampipes-data-stream.ipynb | 70 ++- ...ine-learning-on-a-streampipes-data-stream.ipynb | 484 +++++++++++++++++ .../docs/getting-started/developing.md | 106 ++++ .../docs/getting-started/first-steps.md | 68 +++ .../docs/getting-started/installation.md | 41 -- .../docs/getting-started/quickstart.md | 22 +- streampipes-client-python/docs/index.md | 39 +- .../docs/scripts/gen_ref_pages.py | 8 +- streampipes-client-python/mkdocs.yml | 7 +- streampipes-client-python/setup.py | 8 +- .../__init__.py | 0 .../__version__.py | 2 +- .../client/__init__.py | 0 .../client/client.py | 12 +- .../client/config.py} | 2 +- .../client/credential_provider.py | 0 .../endpoint/__init__.py | 0 .../endpoint/api/__init__.py | 0 .../endpoint/api/data_lake_measure.py | 14 +- .../endpoint/api/data_stream.py | 12 +- .../endpoint/endpoint.py | 36 +- .../endpoint/exceptions.py | 0 .../endpoint/messaging/__init__.py | 0 .../function_zoo}/__init__.py | 0 .../streampipes/function_zoo/river_function.py | 184 +++++++ .../utils => streampipes/functions}/__init__.py | 0 .../functions/broker/__init__.py | 5 +- .../functions/broker/broker.py | 55 +- .../streampipes/functions/broker/broker_handler.py | 53 ++ .../functions/broker/nats_broker.py | 31 +- .../functions/broker/output_collector.py | 65 +++ .../functions/function_handler.py | 57 +- .../functions/registration.py | 2 +- .../functions/streampipes_function.py | 39 +- .../functions/utils}/__init__.py | 0 .../functions/utils/async_iter_handler.py | 0 .../functions/utils/data_stream_context.py | 6 +- .../functions/utils/data_stream_generator.py | 73 +++ .../functions/utils/function_context.py | 4 +- .../model}/__init__.py | 0 .../model/common.py | 68 +-- .../model/container/__init__.py | 0 .../model/container/data_lake_measures.py | 6 +- .../model/container/data_streams.py | 6 +- .../model/container/resource_container.py | 2 +- .../model/resource/__init__.py | 0 .../model/resource/data_lake_measure.py | 4 +- .../model/resource/data_lake_series.py | 2 +- .../model/resource/data_stream.py | 75 ++- .../model/resource/function_definition.py | 35 +- .../model/resource/resource.py | 2 +- .../tests/client/test_client.py | 16 +- .../tests/client/test_credential_provider.py | 2 +- .../tests/client/test_data_lake_series.py | 12 +- .../tests/client/test_endpoint.py | 81 +-- .../tests/functions/test_function_handler.py | 110 +++- .../tests/functions/test_river_function.py | 206 ++++++++ .../model}/__init__.py | 0 .../model/resource}/__init__.py | 0 .../tests/model/resource/test_data_stream.py | 73 +++ streampipes-client/pom.xml | 21 +- .../streampipes/client/api/DataProcessorApi.java | 3 +- .../apache/streampipes/client/api/DataSinkApi.java | 3 +- .../streampipes/client/api/DataStreamApi.java | 3 +- .../apache/streampipes/client/api/PipelineApi.java | 2 +- streampipes-commons/pom.xml | 2 +- .../streampipes/commons/constants/CustomEnvs.java | 20 +- .../commons/constants/DefaultEnvValues.java | 9 +- .../apache/streampipes/commons/constants/Envs.java | 87 ++- .../commons/environment/DefaultEnvironment.java | 80 +++ .../commons/environment/Environment.java | 40 ++ .../commons/environment/Environments.java | 16 +- .../environment/variable/EnvironmentVariable.java | 20 +- .../streampipes/commons/networking/Networking.java | 21 +- streampipes-config/pom.xml | 13 +- .../streampipes/config/backend/BackendConfig.java | 104 +--- .../config/backend/BackendConfigKeys.java | 11 - streampipes-connect-management/pom.xml | 15 +- streampipes-data-explorer-commons/pom.xml | 13 +- streampipes-data-explorer/pom.xml | 13 +- .../dataexplorer/DataLakeManagementV4.java | 10 +- .../dataexplorer/query/DataExplorerQuery.java | 10 +- .../dataexplorer/sdk/DataLakeQueryBuilder.java | 8 +- .../dataexplorer/v4/query/DataExplorerQueryV4.java | 24 +- streampipes-data-export/pom.xml | 10 +- .../export/resolver/AbstractResolver.java | 7 + .../export/resolver/AdapterResolver.java | 3 +- .../export/resolver/DataSourceResolver.java | 3 +- .../export/resolver/PipelineResolver.java | 9 +- .../export/utils/EventGroundingProcessor.java | 60 ++- streampipes-dataformat-cbor/pom.xml | 9 +- streampipes-dataformat-fst/pom.xml | 9 +- streampipes-dataformat-json/pom.xml | 9 +- streampipes-dataformat-smile/pom.xml | 9 +- streampipes-dataformat/pom.xml | 7 +- streampipes-extensions-api/pom.xml | 9 +- streampipes-extensions-management/pom.xml | 37 +- ...StreamPipesClientRuntimeConnectionResolver.java | 21 +- .../management/connect/adapter/Adapter.java | 16 - .../elements/SendToBrokerAdapterSink.java | 15 +- .../elements/SendToJmsAdapterSink.java | 4 +- .../elements/SendToKafkaAdapterSink.java | 6 +- .../elements/SendToMqttAdapterSink.java | 4 +- .../elements/SendToNatsAdapterSink.java | 4 +- streampipes-extensions/.github/workflows/build.yml | 587 --------------------- streampipes-extensions/pom.xml | 6 +- .../streampipes-connect-adapters-iiot/pom.xml | 8 +- .../streampipes-connect-adapters/pom.xml | 6 +- .../streampipes-connectors-influx/pom.xml | 12 +- .../streampipes-extensions-all-iiot/pom.xml | 26 +- .../streampipes-extensions-all-jvm/pom.xml | 12 +- .../pom.xml | 18 +- .../streampipes-pipeline-elements-all-jvm/pom.xml | 28 +- .../pom.xml | 2 +- .../streampipes-pipeline-elements-shared/pom.xml | 10 +- .../pom.xml | 8 +- .../pom.xml | 6 +- .../streampipes-processors-enricher-flink/pom.xml | 6 +- .../streampipes-processors-enricher-jvm/pom.xml | 6 +- .../streampipes-processors-filters-jvm/pom.xml | 6 +- .../processors/filters/jvm/FiltersJvmInit.java | 4 +- .../processor/sdt/SwingingDoorTrendingFilter.java | 182 +++++++ .../sdt/SwingingDoorTrendingFilterProcessor.java | 133 +++++ .../documentation.md | 79 +++ .../icon.png | Bin 0 -> 107607 bytes .../strings.en | 34 ++ .../streampipes-processors-filters-siddhi/pom.xml | 8 +- .../streampipes-processors-geo-flink/pom.xml | 6 +- .../streampipes-processors-geo-jvm/pom.xml | 20 +- .../streampipes/processors/geo/jvm/GeoJvmInit.java | 32 +- .../SpNotSupportedGeometryException.java | 24 +- .../geo/jvm/jts/helper/SpGeometryBuilder.java | 60 ++- .../geo/jvm/jts/helper/SpReprojectionBuilder.java | 304 +++++++++++ .../LatLngToJtsPointProcessor.java | 7 +- .../reprojection/ReprojectionProcessor.java | 146 +++++ .../trajectory/TrajectoryFromPointsProcessor.java | 16 +- .../HaversineDistanceCalculatorProcessor.java | 21 +- .../speedcalculator/SpeedCalculatorProcessor.java | 6 +- .../documentation.md | 56 +- .../icon.png | Bin 0 -> 20720 bytes .../strings.en | 14 +- .../pom.xml | 12 +- .../pom.xml | 29 +- .../pom.xml | 6 +- .../pom.xml | 8 +- .../streampipes-processors-text-mining-jvm/pom.xml | 11 +- .../pom.xml | 10 +- .../pom.xml | 8 +- .../streampipes-sinks-brokers-jvm/pom.xml | 8 +- .../streampipes-sinks-databases-flink/pom.xml | 27 +- .../streampipes-sinks-databases-jvm/pom.xml | 6 +- .../streampipes-sinks-internal-jvm/pom.xml | 10 +- .../streampipes-sinks-notifications-jvm/pom.xml | 8 +- .../streampipes-sources-vehicle-simulator/pom.xml | 10 +- .../simulator/simulator/VehicleDataSimulator.java | 13 +- .../pom.xml | 13 +- .../simulator/utils/WatertankDataSimulator.java | 13 +- streampipes-integration-tests/pom.xml | 8 +- streampipes-logging/pom.xml | 5 +- streampipes-mail/pom.xml | 9 +- streampipes-maven-plugin/pom.xml | 9 +- streampipes-measurement-units/pom.xml | 7 +- streampipes-messaging-jms/pom.xml | 4 +- streampipes-messaging-kafka/pom.xml | 6 +- .../messaging/kafka/SpKafkaProducer.java | 6 +- streampipes-messaging-mqtt/pom.xml | 4 +- streampipes-messaging-nats/pom.xml | 4 +- streampipes-messaging/pom.xml | 4 +- streampipes-model-client/pom.xml | 9 +- streampipes-model-shared/pom.xml | 2 +- streampipes-model/pom.xml | 13 +- .../streampipes/model/datalake/DataSeries.java | 15 + .../{DataSeries.java => DataSeriesBuilder.java} | 52 +- .../model/datalake/SpQueryResultBuilder.java | 73 +++ .../model/datalake/DataSeriesBuilderTest.java | 90 ++++ .../model/datalake/SpQueryResultBuilderTest.java | 116 ++++ streampipes-pipeline-management/pom.xml | 42 +- .../provider/StoredPipelineElementProvider.java | 13 +- .../runtime/PipelineElementRuntimeInfoFetcher.java | 13 +- .../manager/setup/AutoInstallation.java | 56 +- streampipes-platform-services/pom.xml | 15 +- streampipes-resource-management/pom.xml | 11 +- .../AbstractPipelineElementResourceManager.java | 9 +- .../resource/management/UserResourceManager.java | 11 +- streampipes-rest-core-base/pom.xml | 13 +- streampipes-rest-extensions/pom.xml | 10 +- .../pe/InvocablePipelineElementResource.java | 4 +- streampipes-rest-shared/pom.xml | 9 +- streampipes-rest/pom.xml | 42 +- .../streampipes/rest/impl/PipelineResource.java | 37 -- .../org/apache/streampipes/rest/impl/Setup.java | 3 + .../org/apache/streampipes/rest/impl/Version.java | 3 + .../rest/impl/pe/DataProcessorResource.java | 12 +- .../streampipes/rest/impl/pe/DataSinkResource.java | 12 +- .../rest/impl/pe/DataStreamResource.java | 31 +- streampipes-sdk-bundle/pom.xml | 25 +- streampipes-sdk/pom.xml | 11 +- ...AbstractConfigurablePipelineElementBuilder.java | 18 + streampipes-security-jwt/pom.xml | 7 +- .../streampipes/security/jwt/KeyGenerator.java | 5 +- streampipes-serializers-json/pom.xml | 11 +- streampipes-service-base/pom.xml | 6 +- .../resources/{logback-spring.xml => logback.xml} | 0 streampipes-service-core/pom.xml | 17 +- .../service/core/StreamPipesEnvChecker.java | 40 +- .../src/main/resources/application.properties | 2 +- streampipes-service-discovery-api/pom.xml | 2 +- streampipes-service-discovery-consul/pom.xml | 8 +- .../svcdiscovery/consul/ConsulProvider.java | 2 +- streampipes-service-discovery/pom.xml | 6 +- streampipes-service-extensions/pom.xml | 6 +- .../extensions/security/WebSecurityConfig.java | 12 +- .../src/main/resources/application.properties | 2 +- streampipes-sources/pom.xml | 7 +- streampipes-storage-api/pom.xml | 9 +- streampipes-storage-couchdb/pom.xml | 15 +- .../storage/couchdb/utils/CouchDbConfig.java | 55 -- .../streampipes/storage/couchdb/utils/Utils.java | 27 +- streampipes-storage-management/pom.xml | 9 +- streampipes-test-utils/pom.xml | 7 +- streampipes-user-management/pom.xml | 13 +- .../encryption/SecretEncryptionManager.java | 7 +- .../user/management/jwt/JwtTokenProvider.java | 7 +- streampipes-vocabulary/pom.xml | 5 +- streampipes-wrapper-distributed/pom.xml | 13 +- streampipes-wrapper-flink/pom.xml | 7 +- streampipes-wrapper-kafka-streams/pom.xml | 7 +- streampipes-wrapper-python/pom.xml | 5 +- streampipes-wrapper-siddhi/pom.xml | 4 +- streampipes-wrapper-standalone/pom.xml | 7 +- .../standalone/function/StreamPipesFunction.java | 10 +- streampipes-wrapper/pom.xml | 17 +- ui/README.md | 11 +- ui/package-lock.json | 343 ++++-------- ui/package.json | 14 +- .../src/lib/apis/pipeline-element.service.ts | 6 +- .../src/lib/apis/pipeline.service.ts | 12 +- .../assets/dialog/base-asset-links.directive.ts | 2 +- ui/src/app/home/components/status.component.ts | 2 +- ui/src/app/info/about/about.component.html | 23 + .../app/notifications/notifications.component.ts | 2 +- ui/src/app/pipelines/pipelines.component.ts | 2 +- .../token/token-management-settings.component.html | 23 +- .../token/token-management-settings.component.scss | 4 + .../token/token-management-settings.component.ts | 8 + 282 files changed, 4921 insertions(+), 2416 deletions(-) diff --cc installer/k8s/values.yaml index 81fa886e3,07edd3dbf..7a71e2733 --- a/installer/k8s/values.yaml +++ b/installer/k8s/values.yaml @@@ -21,67 -21,30 +21,67 @@@ persistentVolumeReclaimPolicy: "Delete persistentVolumeAccessModes: "ReadWriteOnce" streampipes: - version: "0.91.0-SNAPSHOT" + version: "0.92.0-SNAPSHOT" registry: "apachestreampipes" + core: + appName: "backend" + port: 8030 + persistence: + storageClassName: "" + claimName: "backend-pvc" + pvName: "backend-pv" + service: + name: "backend" + port: 8030 + ui: + appName: "ui" + service: + type: "LoadBalancer" + nodePort: 8088 + port: 8088 + port: 8088 external: - consulVersion: 1.14.3 - couchdbVersion: 3.3.1 - flinkVersion: 1.13.5-scala_2.11 + consul: + appName: "consul" + version: 1.14.3 + persistence: + storageClassName: "" + claimName: "consul-pvc" + pvName: "consul-pv" + couchdb: + appName: "couchdb" + version: 3.3.1 + user: "admin" + password: "admin" + persistence: + storageClassName: "" + claimName: "couchdb-pvc" + pvName: "couchdb-pv" + influxdb: + appName: "influxdb" + version: 2.6 + username: "admin" + password: "sp-admin" + adminToken: "sp-admin" + initOrg: "sp" + initBucket: "sp" + # For database migration in v0.91.0 - set init mode to 'upgrade' to migrate an existing installation + initMode: "setup" + persistence: + storageClassName: "" + claimName: "influxdb2-pvc" + claimNameV1: "influxdb-pvc" + pvName: "influxdb2-pv" + pvNameV1: "influxdb-pv" + nats: + appName: "nats" + service: + type: "LoadBalancer" + externalTrafficPolicy: "Local" + version: kafkaVersion: 2.2.0 zookeeperVersion: 3.4.13 - influxdbVersion: 2.6 -# For database migration in v0.91.0 - set init mode to 'upgrade' to migrate an existing installation -influxdb: - username: "admin" - password: "sp-admin" - adminToken: "sp-admin" - initOrg: "sp" - initBucket: "sp" - initMode: "setup" - - -# NOTE: Local testing only! -# minikube start --mount-string ${HOME}/streampipes-k8s:/streampipes-k8s --mount --memory=4g --cpus=4 -# Specify minikube directory: -hostPath: "/streampipes-k8s"
