This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
from c8ed11df [FLINK-28875] Add FlinkSessionJobControllerTest
new 8d285fe1 [FLINK-30260][autoscaler] Define scaling metrics
new 5aa9956e [FLINK-30260][autoscaler] Utilities for collecting and
computing metrics
new d535333a [FLINK-30260][autoscaler] Collect and compute scaling metrics
through Flink REST API
new 0b3cc131 [FLINK-30260][autoscaler] Add scaling metric evaluation logic
new 37bf5107 [FLINK-30260][autoscaler] Add scaling execution logic
new 53b1639a [FLINK-30260][autoscaler] Add JobAutoScaler component and
configs
new 4408c665 [FLINK-30260][autoscaler] Integrate autoscaler components
with reconciler mechanism
new c5e437f3 [FLINK-30260][autoscaler] Autoscaler example application
new 86458aac [docs][autoscaler] Add docs page + enable config doc
generation for Autoscaler options
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 2 +-
docs/content/docs/custom-resource/autoscaler.md | 158 ++++++++
docs/content/docs/custom-resource/reference.md | 1 +
docs/content/docs/operations/configuration.md | 6 +
.../generated/auto_scaler_configuration.html | 90 +++++
.../kustomization.yaml => autoscaling/Dockerfile} | 9 +-
.../autoscaling.yaml} | 37 +-
.../pom.xml | 91 ++++-
.../main/java/autoscaling/AutoscalingExample.java | 40 ++
.../src/main/resources/log4j2.properties | 8 +-
.../configuration/ConfigOptionsDocGenerator.java | 6 +-
.../kubernetes/operator/api/spec/FlinkVersion.java | 3 +-
.../flink/kubernetes/operator/FlinkOperator.java | 14 +-
.../operator/autoscaler/AutoScalerInfo.java | 189 +++++++++
.../Cleanup.java} | 18 +-
.../operator/autoscaler/JobAutoScaler.java | 204 ++++++++++
.../autoscaler/RestApiMetricsCollector.java | 100 +++++
.../operator/autoscaler/ScalingExecutor.java | 362 +++++++++++++++++
.../autoscaler/ScalingMetricCollector.java | 443 +++++++++++++++++++++
.../autoscaler/ScalingMetricEvaluator.java | 228 +++++++++++
.../operator/autoscaler/ScalingSummary.java | 51 +++
.../autoscaler/config/AutoScalerOptions.java | 121 ++++++
.../autoscaler/metrics/CollectedMetrics.java | 25 +-
.../autoscaler/metrics/EvaluatedScalingMetric.java | 21 +-
.../operator/autoscaler/metrics/FlinkMetric.java | 49 +++
.../operator/autoscaler/metrics/ScalingMetric.java | 74 ++++
.../autoscaler/metrics/ScalingMetrics.java | 159 ++++++++
.../operator/autoscaler/topology/JobTopology.java | 154 +++++++
.../topology/VertexInfo.java} | 22 +-
.../operator/autoscaler/utils/AutoScalerUtils.java | 70 ++++
.../autoscaler/utils/JobVertexSerDeModule.java | 54 +++
.../AbstractFlinkResourceReconciler.java | 13 +-
.../deployment/AbstractJobReconciler.java | 6 +-
.../deployment/ApplicationReconciler.java | 6 +-
.../reconciler/deployment/ReconcilerFactory.java | 12 +-
.../reconciler/deployment/SessionReconciler.java | 6 +-
.../sessionjob/SessionJobReconciler.java | 6 +-
.../operator/service/AbstractFlinkService.java | 4 +-
.../kubernetes/operator/service/FlinkService.java | 3 +
.../operator/utils/KubernetesClientUtils.java | 19 +
.../rest/messages/job/metrics/IOMetricsInfo.java | 191 +++++++++
.../kubernetes/operator/TestingFlinkService.java | 2 +-
.../autoscaler/BacklogBasedScalingTest.java | 352 ++++++++++++++++
.../operator/autoscaler/JobTopologyTest.java | 99 +++++
.../MetricsCollectionAndEvaluationTest.java | 251 ++++++++++++
.../operator/autoscaler/ScalingExecutorTest.java | 350 ++++++++++++++++
.../autoscaler/ScalingMetricEvaluatorTest.java | 227 +++++++++++
.../autoscaler/TestingMetricsCollector.java | 79 ++++
.../autoscaler/metrics/ScalingMetricsTest.java | 178 +++++++++
.../TestingFlinkDeploymentController.java | 5 +-
.../TestingFlinkSessionJobController.java | 5 +-
.../sessionjob/FlinkSessionJobObserverTest.java | 3 +-
.../deployment/ApplicationReconcilerTest.java | 3 +-
.../ApplicationReconcilerUpgradeModeTest.java | 3 +-
.../deployment/SessionReconcilerTest.java | 9 +-
.../sessionjob/SessionJobReconcilerTest.java | 12 +-
.../operator/service/NativeFlinkServiceTest.java | 2 +-
.../crds/flinkdeployments.flink.apache.org-v1.yml | 1 +
pom.xml | 1 +
59 files changed, 4538 insertions(+), 119 deletions(-)
create mode 100644 docs/content/docs/custom-resource/autoscaler.md
create mode 100644
docs/layouts/shortcodes/generated/auto_scaler_configuration.html
copy examples/{kustomize/sidecar/kustomization.yaml => autoscaling/Dockerfile}
(87%)
copy examples/{basic-checkpoint-ha.yaml => autoscaling/autoscaling.yaml} (69%)
copy examples/{flink-sql-runner-example => autoscaling}/pom.xml (57%)
create mode 100644
examples/autoscaling/src/main/java/autoscaling/AutoscalingExample.java
copy {flink-kubernetes-webhook =>
examples/autoscaling}/src/main/resources/log4j2.properties (86%)
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/AutoScalerInfo.java
copy
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/{exception/StatusConflictException.java
=> autoscaler/Cleanup.java} (68%)
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/JobAutoScaler.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/RestApiMetricsCollector.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutor.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricCollector.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricEvaluator.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingSummary.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/config/AutoScalerOptions.java
copy
flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/spec/JobState.java
=>
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/CollectedMetrics.java
(62%)
copy
flink-kubernetes-operator-api/src/main/java/org/apache/flink/kubernetes/operator/api/spec/Resource.java
=>
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/EvaluatedScalingMetric.java
(73%)
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/FlinkMetric.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetric.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetrics.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/topology/JobTopology.java
copy
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/{metrics/CustomResourceMetrics.java
=> autoscaler/topology/VertexInfo.java} (71%)
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/utils/AutoScalerUtils.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/kubernetes/operator/autoscaler/utils/JobVertexSerDeModule.java
create mode 100644
flink-kubernetes-operator/src/main/java/org/apache/flink/runtime/rest/messages/job/metrics/IOMetricsInfo.java
create mode 100644
flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/BacklogBasedScalingTest.java
create mode 100644
flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/JobTopologyTest.java
create mode 100644
flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/MetricsCollectionAndEvaluationTest.java
create mode 100644
flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingExecutorTest.java
create mode 100644
flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/ScalingMetricEvaluatorTest.java
create mode 100644
flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/TestingMetricsCollector.java
create mode 100644
flink-kubernetes-operator/src/test/java/org/apache/flink/kubernetes/operator/autoscaler/metrics/ScalingMetricsTest.java
