(flink) 01/02: [FLINK-34052][examples] Install the shaded streaming example jars in the maven repo
This is an automated email from the ASF dual-hosted git repository. guoweijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git commit 300381c2b4bb7e9c711459fb2ad55dbaf5add7ae Author: Zhanghao Chen AuthorDate: Sun Jan 28 00:08:07 2024 +0800 [FLINK-34052][examples] Install the shaded streaming example jars in the maven repo --- flink-examples/flink-examples-streaming/pom.xml | 6 ++ 1 file changed, 6 insertions(+) diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml index 582fb28ee29..3a5459bd1ad 100644 --- a/flink-examples/flink-examples-streaming/pom.xml +++ b/flink-examples/flink-examples-streaming/pom.xml @@ -318,6 +318,8 @@ under the License. false Iteration + true + Iteration org.apache.flink:flink-connector-datagen @@ -352,6 +354,8 @@ under the License. false TopSpeedWindowing + true + TopSpeedWindowing org.apache.flink:flink-connector-datagen @@ -389,6 +393,8 @@ under the License. false SessionWindowing + true + SessionWindowing org.apache.flink:flink-connector-datagen
(flink) branch master updated (20450485b20 -> 324a5e45c80)
This is an automated email from the ASF dual-hosted git repository. guoweijie pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 20450485b20 [FLINK-34222][table-planner] Supports mini-batch for streaming regular join new 300381c2b4b [FLINK-34052][examples] Install the shaded streaming example jars in the maven repo new 324a5e45c80 [hotfix][examples] Doc on why MatrixVectorMul example is not bundled with Flink dist nor deployed in Maven repo The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: flink-examples/flink-examples-streaming/pom.xml | 10 +- 1 file changed, 9 insertions(+), 1 deletion(-)
(flink) 02/02: [hotfix][examples] Doc on why MatrixVectorMul example is not bundled with Flink dist nor deployed in Maven repo
This is an automated email from the ASF dual-hosted git repository. guoweijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git commit 324a5e45c80464335a95cad7fbccfd531d0b098b Author: Zhanghao Chen AuthorDate: Sun Jan 28 13:14:19 2024 +0800 [hotfix][examples] Doc on why MatrixVectorMul example is not bundled with Flink dist nor deployed in Maven repo --- flink-examples/flink-examples-streaming/pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml index 3a5459bd1ad..04ad7e1adde 100644 --- a/flink-examples/flink-examples-streaming/pom.xml +++ b/flink-examples/flink-examples-streaming/pom.xml @@ -274,7 +274,9 @@ under the License. false - + MatrixVectorMul package
(flink) branch master updated: [FLINK-34222][table-planner] Supports mini-batch for streaming regular join
This is an automated email from the ASF dual-hosted git repository.
ron pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 20450485b20 [FLINK-34222][table-planner] Supports mini-batch for
streaming regular join
20450485b20 is described below
commit 20450485b20cb213b96318b0c3275e42c0300e15
Author: yeming
AuthorDate: Mon Jan 22 10:26:26 2024 +0800
[FLINK-34222][table-planner] Supports mini-batch for streaming regular join
This closes #24161.
---
.../exec/stream/StreamExecChangelogNormalize.java | 8 +--
.../stream/StreamExecGlobalGroupAggregate.java | 7 +--
.../exec/stream/StreamExecGroupAggregate.java | 7 +--
.../StreamExecIncrementalGroupAggregate.java | 3 +-
.../plan/nodes/exec/stream/StreamExecJoin.java | 44 +
.../exec/stream/StreamExecLocalGroupAggregate.java | 3 +-
.../table/planner/plan/utils/MinibatchUtil.java| 73 ++
.../nodes/physical/stream/StreamPhysicalJoin.scala | 3 +-
.../table/planner/plan/utils/AggregateUtil.scala | 10 ---
.../analyze/GroupAggregationAnalyzerTest.xml | 4 +-
.../plan/stream/sql/MiniBatchIntervalInferTest.xml | 2 +-
.../planner/plan/stream/sql/join/JoinTest.xml | 4 +-
.../planner/runtime/stream/sql/JoinITCase.scala| 4 +-
.../planner/runtime/stream/table/JoinITCase.scala | 4 +-
14 files changed, 131 insertions(+), 45 deletions(-)
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecChangelogNormalize.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecChangelogNormalize.java
index 710eac5eead..29171195e85 100644
---
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecChangelogNormalize.java
+++
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecChangelogNormalize.java
@@ -25,7 +25,6 @@ import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.streaming.api.operators.KeyedProcessOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.transformations.OneInputTransformation;
-import org.apache.flink.table.api.config.ExecutionConfigOptions;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.planner.codegen.EqualiserCodeGenerator;
import org.apache.flink.table.planner.delegation.PlannerBase;
@@ -39,8 +38,8 @@ import
org.apache.flink.table.planner.plan.nodes.exec.InputProperty;
import
org.apache.flink.table.planner.plan.nodes.exec.SingleTransformationTranslator;
import org.apache.flink.table.planner.plan.nodes.exec.StateMetadata;
import org.apache.flink.table.planner.plan.nodes.exec.utils.ExecNodeUtil;
-import org.apache.flink.table.planner.plan.utils.AggregateUtil;
import org.apache.flink.table.planner.plan.utils.KeySelectorUtil;
+import org.apache.flink.table.planner.plan.utils.MinibatchUtil;
import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
import org.apache.flink.table.runtime.keyselector.RowDataKeySelector;
import org.apache.flink.table.runtime.operators.bundle.KeyedMapBundleOperator;
@@ -145,8 +144,7 @@ public class StreamExecChangelogNormalize extends
ExecNodeBase
final long stateRetentionTime =
StateMetadata.getStateTtlForOneInputOperator(config,
stateMetadataList);
-final boolean isMiniBatchEnabled =
-
config.get(ExecutionConfigOptions.TABLE_EXEC_MINIBATCH_ENABLED);
+final boolean isMiniBatchEnabled =
MinibatchUtil.isMiniBatchEnabled(config);
GeneratedRecordEqualiser generatedEqualiser =
new EqualiserCodeGenerator(
@@ -166,7 +164,7 @@ public class StreamExecChangelogNormalize extends
ExecNodeBase
true, // generateInsert
false, // inputInsertOnly
generatedEqualiser);
-CountBundleTrigger trigger =
AggregateUtil.createMiniBatchTrigger(config);
+CountBundleTrigger trigger =
MinibatchUtil.createMiniBatchTrigger(config);
operator = new KeyedMapBundleOperator<>(processFunction, trigger);
} else {
ProcTimeDeduplicateKeepLastRowFunction processFunction =
diff --git
a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecGlobalGroupAggregate.java
b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/stream/StreamExecGlobalGroupAggregate.java
index ed82380a195..37809b9cc92 100644
---
(flink) branch master updated: [FLINK-34249][runtime] Remove DefaultSlotTracker related logic.
This is an automated email from the ASF dual-hosted git repository.
fanrui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new a24f7717847 [FLINK-34249][runtime] Remove DefaultSlotTracker related
logic.
a24f7717847 is described below
commit a24f7717847ce4e4c511070257e99d7c3f948d2a
Author: Roc Marshal
AuthorDate: Wed Jan 24 20:25:55 2024 +0800
[FLINK-34249][runtime] Remove DefaultSlotTracker related logic.
---
.../slotmanager/DeclarativeTaskManagerSlot.java| 146 -
.../slotmanager/DefaultSlotTracker.java| 337 -
.../resourcemanager/slotmanager/SlotTracker.java | 112 ---
.../slotmanager/DefaultSlotTrackerTest.java| 332
.../slotmanager/SlotStatusReconcilerTest.java | 243 ---
5 files changed, 1170 deletions(-)
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DeclarativeTaskManagerSlot.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DeclarativeTaskManagerSlot.java
deleted file mode 100644
index 6f730d6c780..000
---
a/flink-runtime/src/main/java/org/apache/flink/runtime/resourcemanager/slotmanager/DeclarativeTaskManagerSlot.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.runtime.resourcemanager.slotmanager;
-
-import org.apache.flink.api.common.JobID;
-import org.apache.flink.runtime.clusterframework.types.AllocationID;
-import org.apache.flink.runtime.clusterframework.types.ResourceProfile;
-import org.apache.flink.runtime.clusterframework.types.SlotID;
-import org.apache.flink.runtime.instance.InstanceID;
-import
org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection;
-import org.apache.flink.util.Preconditions;
-
-import javax.annotation.Nullable;
-
-/**
- * A DeclarativeTaskManagerSlot represents a slot located in a TaskExecutor.
It contains the
- * necessary information for initiating the allocation of the slot, and keeps
track of the state of
- * the slot.
- */
-class DeclarativeTaskManagerSlot implements TaskManagerSlotInformation {
-
-/** The unique identification of this slot. */
-private final SlotID slotId;
-
-/** The resource profile of this slot. */
-private final ResourceProfile resourceProfile;
-
-/** Gateway to the TaskExecutor which owns the slot. */
-private final TaskExecutorConnection taskManagerConnection;
-
-/** Job id for which this slot has been allocated. */
-@Nullable private JobID jobId;
-
-private SlotState state = SlotState.FREE;
-
-private long allocationStartTimeStamp;
-
-public DeclarativeTaskManagerSlot(
-SlotID slotId,
-ResourceProfile resourceProfile,
-TaskExecutorConnection taskManagerConnection) {
-this.slotId = slotId;
-this.resourceProfile = resourceProfile;
-this.taskManagerConnection = taskManagerConnection;
-}
-
-@Override
-public SlotState getState() {
-return state;
-}
-
-@Override
-public SlotID getSlotId() {
-return slotId;
-}
-
-@Override
-public AllocationID getAllocationId() {
-throw new UnsupportedOperationException();
-}
-
-@Override
-public ResourceProfile getResourceProfile() {
-return resourceProfile;
-}
-
-@Override
-public TaskExecutorConnection getTaskManagerConnection() {
-return taskManagerConnection;
-}
-
-@Nullable
-@Override
-public JobID getJobId() {
-return jobId;
-}
-
-@Override
-public InstanceID getInstanceId() {
-return taskManagerConnection.getInstanceID();
-}
-
-public long getAllocationStartTimestamp() {
-return allocationStartTimeStamp;
-}
-
-public void startAllocation(JobID jobId) {
-Preconditions.checkState(
-state == SlotState.FREE, "Slot must be free to be assigned a
slot request.");
-
-this.jobId = jobId;
-this.state = SlotState.PENDING;
-
(flink-connector-mongodb) branch main updated: [hotfix][test][connectors/mongodb] Update MongoWriterITCase to be compatible with updated SinkV2 interfaces
This is an automated email from the ASF dual-hosted git repository.
leonard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-connector-mongodb.git
The following commit(s) were added to refs/heads/main by this push:
new e1b9684 [hotfix][test][connectors/mongodb] Update MongoWriterITCase
to be compatible with updated SinkV2 interfaces
e1b9684 is described below
commit e1b96849dc314b8b31a460757168972f17fbec1a
Author: Jiabao Sun
AuthorDate: Mon Jan 29 09:50:00 2024 +0800
[hotfix][test][connectors/mongodb] Update MongoWriterITCase to be
compatible with updated SinkV2 interfaces
This closes #22.
---
.../mongodb/sink/writer/MongoWriterITCase.java | 33 +++---
1 file changed, 16 insertions(+), 17 deletions(-)
diff --git
a/flink-connector-mongodb/src/test/java/org/apache/flink/connector/mongodb/sink/writer/MongoWriterITCase.java
b/flink-connector-mongodb/src/test/java/org/apache/flink/connector/mongodb/sink/writer/MongoWriterITCase.java
index 84767e8..bd3ca66 100644
---
a/flink-connector-mongodb/src/test/java/org/apache/flink/connector/mongodb/sink/writer/MongoWriterITCase.java
+++
b/flink-connector-mongodb/src/test/java/org/apache/flink/connector/mongodb/sink/writer/MongoWriterITCase.java
@@ -17,8 +17,9 @@
package org.apache.flink.connector.mongodb.sink.writer;
+import org.apache.flink.connector.base.DeliveryGuarantee;
import org.apache.flink.connector.base.sink.writer.TestSinkInitContext;
-import org.apache.flink.connector.mongodb.common.config.MongoConnectionOptions;
+import org.apache.flink.connector.mongodb.sink.MongoSink;
import org.apache.flink.connector.mongodb.sink.config.MongoWriteOptions;
import org.apache.flink.connector.mongodb.sink.writer.context.MongoSinkContext;
import
org.apache.flink.connector.mongodb.sink.writer.serializer.MongoSerializationSchema;
@@ -49,6 +50,7 @@ import org.testcontainers.containers.MongoDBContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
+import java.io.IOException;
import java.util.Optional;
import static
org.apache.flink.connector.mongodb.testutils.MongoTestUtil.assertThatIdsAreNotWritten;
@@ -235,7 +237,7 @@ public class MongoWriterITCase {
MongoWriteOptions.builder()
.setBatchSize(batchSize)
.setBatchIntervalMs(batchIntervalMs)
-.setMaxRetries(0)
+.setDeliveryGuarantee(DeliveryGuarantee.NONE)
.build();
MongoSerializationSchema testSerializationSchema =
@@ -269,7 +271,8 @@ public class MongoWriterITCase {
}
private static MongoWriter createWriter(
-String collection, int batchSize, long batchIntervalMs, boolean
flushOnCheckpoint) {
+String collection, int batchSize, long batchIntervalMs, boolean
flushOnCheckpoint)
+throws IOException {
return createWriter(
collection,
batchSize,
@@ -283,28 +286,24 @@ public class MongoWriterITCase {
int batchSize,
long batchIntervalMs,
boolean flushOnCheckpoint,
-MongoSerializationSchema serializationSchema) {
+MongoSerializationSchema serializationSchema)
+throws IOException {
-MongoConnectionOptions connectionOptions =
-MongoConnectionOptions.builder()
+MongoSink mongoSink =
+MongoSink.builder()
.setUri(MONGO_CONTAINER.getConnectionString())
.setDatabase(TEST_DATABASE)
.setCollection(collection)
-.build();
-
-MongoWriteOptions writeOptions =
-MongoWriteOptions.builder()
.setBatchSize(batchSize)
.setBatchIntervalMs(batchIntervalMs)
-.setMaxRetries(0)
+.setDeliveryGuarantee(
+flushOnCheckpoint
+? DeliveryGuarantee.AT_LEAST_ONCE
+: DeliveryGuarantee.NONE)
+.setSerializationSchema(serializationSchema)
.build();
-return new MongoWriter<>(
-connectionOptions,
-writeOptions,
-flushOnCheckpoint,
-sinkInitContext,
-serializationSchema);
+return (MongoWriter) mongoSink.createWriter(sinkInitContext);
}
private static Document buildMessage(int id) {
(flink) branch master updated (2e56caf11c0 -> ddbf87f2a7a)
This is an automated email from the ASF dual-hosted git repository. zhuzh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/flink.git from 2e56caf11c0 [FLINK-34122][core] Deprecate old serialization config methods and options add ddbf87f2a7a [FLINK-34245][python] Fix config retrieval logic from nested YAML in pyflink_gateway_server with flattened keys. No new revisions were added by this update. Summary of changes: flink-python/pyflink/pyflink_gateway_server.py | 15 ++- 1 file changed, 14 insertions(+), 1 deletion(-)
