dannycranmer commented on a change in pull request #18669: URL: https://github.com/apache/flink/pull/18669#discussion_r803439459
########## File path: flink-connectors/flink-connector-aws-kinesis-data-streams/src/main/java/org/apache/flink/connector/kinesis/sink/KinesisDataStreamsStateSerializer.java ########## @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.kinesis.sink; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.connector.base.sink.writer.AsyncSinkWriterStateSerializer; + +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; + +import javax.annotation.Nullable; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +/** Kinesis Streams implementation {@link AsyncSinkWriterStateSerializer}. */ +@Internal +public class KinesisDataStreamsStateSerializer + extends AsyncSinkWriterStateSerializer<PutRecordsRequestEntry> { + @Override + protected void serializeRequestToStream(PutRecordsRequestEntry request, DataOutputStream out) + throws IOException { + out.write(request.data().asByteArrayUnsafe()); + serializePartitionKeyToStream(request.partitionKey(), out); + serializeExplicitHashKeyToStream(request.explicitHashKey(), out); Review comment: We never set this in the `ElementConverter` right? Should we remove this from here and assert that it is null? ########## File path: flink-connectors/flink-connector-aws-kinesis-data-streams/src/main/java/org/apache/flink/connector/kinesis/sink/KinesisDataStreamsStateSerializer.java ########## @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.kinesis.sink; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.connector.base.sink.writer.AsyncSinkWriterStateSerializer; + +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; + +import javax.annotation.Nullable; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +/** Kinesis Streams implementation {@link AsyncSinkWriterStateSerializer}. */ +@Internal +public class KinesisDataStreamsStateSerializer + extends AsyncSinkWriterStateSerializer<PutRecordsRequestEntry> { + @Override + protected void serializeRequestToStream(PutRecordsRequestEntry request, DataOutputStream out) + throws IOException { + out.write(request.data().asByteArrayUnsafe()); + serializePartitionKeyToStream(request.partitionKey(), out); + serializeExplicitHashKeyToStream(request.explicitHashKey(), out); + } + + protected void serializePartitionKeyToStream(String partitionKey, DataOutputStream out) + throws IOException { + out.writeInt(partitionKey.length()); + out.write(partitionKey.getBytes()); + } + + protected void serializeExplicitHashKeyToStream( + @Nullable String explicitHashKey, DataOutputStream out) throws IOException { + if (explicitHashKey == null) { + out.writeInt(0); + return; + } + + out.writeInt(explicitHashKey.length()); + out.write(explicitHashKey.getBytes()); + } Review comment: These methods are duplicated/could be combined. ########## File path: flink-connectors/flink-connector-aws-kinesis-data-streams/src/main/java/org/apache/flink/connector/kinesis/sink/KinesisDataStreamsStateSerializer.java ########## @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.connector.kinesis.sink; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.connector.base.sink.writer.AsyncSinkWriterStateSerializer; + +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry; + +import javax.annotation.Nullable; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +/** Kinesis Streams implementation {@link AsyncSinkWriterStateSerializer}. */ +@Internal +public class KinesisDataStreamsStateSerializer + extends AsyncSinkWriterStateSerializer<PutRecordsRequestEntry> { + @Override + protected void serializeRequestToStream(PutRecordsRequestEntry request, DataOutputStream out) + throws IOException { + out.write(request.data().asByteArrayUnsafe()); + serializePartitionKeyToStream(request.partitionKey(), out); + serializeExplicitHashKeyToStream(request.explicitHashKey(), out); + } + + protected void serializePartitionKeyToStream(String partitionKey, DataOutputStream out) + throws IOException { + out.writeInt(partitionKey.length()); + out.write(partitionKey.getBytes()); + } + + protected void serializeExplicitHashKeyToStream( + @Nullable String explicitHashKey, DataOutputStream out) throws IOException { + if (explicitHashKey == null) { + out.writeInt(0); + return; + } + + out.writeInt(explicitHashKey.length()); + out.write(explicitHashKey.getBytes()); + } + + @Override + protected PutRecordsRequestEntry deserializeRequestFromStream( + long requestSize, DataInputStream in) throws IOException { + byte[] requestData = readBytes(in, (int) requestSize); + + return PutRecordsRequestEntry.builder() + .data(SdkBytes.fromByteArray(requestData)) + .partitionKey(deserializePartitionKeyToStream(in)) + .explicitHashKey(deserializeExplicitHashKeyToStream(in)) + .build(); + } + + protected String deserializePartitionKeyToStream(DataInputStream in) throws IOException { + int partitionKeyLength = readInt(in); + byte[] requestPartitionKeyData = readBytes(in, partitionKeyLength); + return new String(requestPartitionKeyData); + } + + protected String deserializeExplicitHashKeyToStream(DataInputStream in) throws IOException { + int explicitHashKeyLength = readInt(in); + if (explicitHashKeyLength == 0) { + return null; + } + + byte[] explicitHashKeyData = readBytes(in, explicitHashKeyLength); + return new String(explicitHashKeyData); + } Review comment: These methods are duplicated/could be combined. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
