This is an automated email from the ASF dual-hosted git repository. tzulitai pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink-statefun.git
commit 3a124bb91f431f79b8bab6b0de831056014ccdd9 Author: Igal Shilman <[email protected]> AuthorDate: Wed Jan 27 18:52:15 2021 +0100 [FLINK-21154] Move *-egress.proto to the sdk protos --- statefun-flink/statefun-flink-core/pom.xml | 2 +- statefun-flink/statefun-flink-io/pom.xml | 63 ++++++++++++ statefun-python-sdk/build-distribution.sh | 6 +- statefun-python-sdk/statefun/kafka_egress_pb2.py | 100 ------------------- statefun-python-sdk/statefun/kinesis_egress_pb2.py | 107 --------------------- .../src/main/protobuf/io}/kafka-egress.proto | 0 .../src/main/protobuf/io}/kinesis-egress.proto | 0 .../main/protobuf/{ => sdk}/request-reply.proto | 0 8 files changed, 66 insertions(+), 212 deletions(-) diff --git a/statefun-flink/statefun-flink-core/pom.xml b/statefun-flink/statefun-flink-core/pom.xml index 05c9f7a..20f56d9 100644 --- a/statefun-flink/statefun-flink-core/pom.xml +++ b/statefun-flink/statefun-flink-core/pom.xml @@ -163,7 +163,7 @@ under the License. <version>${project.version}</version> <type>jar</type> <outputDirectory>${additional-sources.dir}</outputDirectory> - <includes>*.proto</includes> + <includes>sdk/*.proto</includes> </artifactItem> </artifactItems> </configuration> diff --git a/statefun-flink/statefun-flink-io/pom.xml b/statefun-flink/statefun-flink-io/pom.xml index e723c61..e0da70e 100644 --- a/statefun-flink/statefun-flink-io/pom.xml +++ b/statefun-flink/statefun-flink-io/pom.xml @@ -27,6 +27,10 @@ under the License. <relativePath>..</relativePath> </parent> + <properties> + <additional-sources.dir>target/additional-sources</additional-sources.dir> + </properties> + <artifactId>statefun-flink-io</artifactId> <dependencies> @@ -37,6 +41,11 @@ under the License. </dependency> <dependency> <groupId>org.apache.flink</groupId> + <artifactId>statefun-sdk-protos</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> <artifactId>flink-streaming-java_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> @@ -48,12 +57,66 @@ under the License. <build> <plugins> + <!-- + The following plugin is executed in the generated-sources phase, + and is responsible to extract the additional *.proto files located + at statefun-sdk-protos.jar. + --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>unpack</id> + <phase>generate-sources</phase> + <goals> + <goal>unpack</goal> + </goals> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.apache.flink</groupId> + <artifactId>statefun-sdk-protos</artifactId> + <version>${project.version}</version> + <type>jar</type> + <outputDirectory>${additional-sources.dir}</outputDirectory> + <includes>io/*.proto</includes> + </artifactItem> + </artifactItems> + </configuration> + </execution> + </executions> + </plugin> + <!-- + The following plugin invokes protoc to generate Java classes out of the *.proto + definitions located at: (1) src/main/protobuf (2) ${additional-sources.dir}. + --> <plugin> <groupId>com.github.os72</groupId> <artifactId>protoc-jar-maven-plugin</artifactId> <version>${protoc-jar-maven-plugin.version}</version> + <executions> + <execution> + <id>generate-protobuf-sources</id> + <phase>generate-sources</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <includeStdTypes>true</includeStdTypes> + <protocVersion>${protobuf.version}</protocVersion> + <cleanOutputFolder>true</cleanOutputFolder> + <inputDirectories> + <inputDirectory>src/main/protobuf</inputDirectory> + <inputDirectory>${additional-sources.dir}</inputDirectory> + </inputDirectories> + <outputDirectory>${basedir}/target/generated-sources/protoc-jar</outputDirectory> + </configuration> + </execution> + </executions> </plugin> </plugins> </build> + </project> \ No newline at end of file diff --git a/statefun-python-sdk/build-distribution.sh b/statefun-python-sdk/build-distribution.sh index 89e564b..3f74e35 100755 --- a/statefun-python-sdk/build-distribution.sh +++ b/statefun-python-sdk/build-distribution.sh @@ -41,15 +41,13 @@ mkdir -p target/ # copy all the sources into target rsync -a --exclude=target * target/ -# copy the addtional .proto files from the SDK -rsync -a ${SDK_PROTOS_DIR}/* target/ +# copy the additional .proto files from the SDK +find ${SDK_PROTOS_DIR} -type f -name "*proto" -exec cp {} target/ \; cd target/ # built the Python SDK inside a Docker container. # This build step also generates Protobuf files. - - docker run -v "${BASE_DIR}/target:/app" \ --rm \ --workdir /app \ diff --git a/statefun-python-sdk/statefun/kafka_egress_pb2.py b/statefun-python-sdk/statefun/kafka_egress_pb2.py deleted file mode 100644 index 0a34749..0000000 --- a/statefun-python-sdk/statefun/kafka_egress_pb2.py +++ /dev/null @@ -1,100 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: kafka-egress.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='kafka-egress.proto', - package='org.apache.flink.statefun.flink.io', - syntax='proto3', - serialized_options=b'\n,org.apache.flink.statefun.flink.io.generatedP\001', - serialized_pb=b'\n\x12kafka-egress.proto\x12\"org.apache.flink.statefun.flink.io\"F\n\x13KafkaProducerRecord\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\r\n\x05topic\x18\x03 \x01(\tB0\n,org.apache.flink.statefun.flink.io.generatedP\x01\x62\x06proto3' -) - - - - -_KAFKAPRODUCERRECORD = _descriptor.Descriptor( - name='KafkaProducerRecord', - full_name='org.apache.flink.statefun.flink.io.KafkaProducerRecord', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='org.apache.flink.statefun.flink.io.KafkaProducerRecord.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value_bytes', full_name='org.apache.flink.statefun.flink.io.KafkaProducerRecord.value_bytes', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='topic', full_name='org.apache.flink.statefun.flink.io.KafkaProducerRecord.topic', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=58, - serialized_end=128, -) - -DESCRIPTOR.message_types_by_name['KafkaProducerRecord'] = _KAFKAPRODUCERRECORD -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -KafkaProducerRecord = _reflection.GeneratedProtocolMessageType('KafkaProducerRecord', (_message.Message,), { - 'DESCRIPTOR' : _KAFKAPRODUCERRECORD, - '__module__' : 'kafka_egress_pb2' - # @@protoc_insertion_point(class_scope:org.apache.flink.statefun.flink.io.KafkaProducerRecord) - }) -_sym_db.RegisterMessage(KafkaProducerRecord) - - -DESCRIPTOR._options = None -# @@protoc_insertion_point(module_scope) diff --git a/statefun-python-sdk/statefun/kinesis_egress_pb2.py b/statefun-python-sdk/statefun/kinesis_egress_pb2.py deleted file mode 100644 index 0a5b5ac..0000000 --- a/statefun-python-sdk/statefun/kinesis_egress_pb2.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: kinesis-egress.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='kinesis-egress.proto', - package='org.apache.flink.statefun.flink.io', - syntax='proto3', - serialized_options=b'\n,org.apache.flink.statefun.flink.io.generatedP\001', - serialized_pb=b'\n\x14kinesis-egress.proto\x12\"org.apache.flink.statefun.flink.io\"l\n\x13KinesisEgressRecord\x12\x15\n\rpartition_key\x18\x01 \x01(\t\x12\x13\n\x0bvalue_bytes\x18\x02 \x01(\x0c\x12\x0e\n\x06stream\x18\x03 \x01(\t\x12\x19\n\x11\x65xplicit_hash_key\x18\x04 \x01(\tB0\n,org.apache.flink.statefun.flink.io.generatedP\x01\x62\x06proto3' -) - - - - -_KINESISEGRESSRECORD = _descriptor.Descriptor( - name='KinesisEgressRecord', - full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='partition_key', full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord.partition_key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value_bytes', full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord.value_bytes', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='stream', full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord.stream', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='explicit_hash_key', full_name='org.apache.flink.statefun.flink.io.KinesisEgressRecord.explicit_hash_key', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=60, - serialized_end=168, -) - -DESCRIPTOR.message_types_by_name['KinesisEgressRecord'] = _KINESISEGRESSRECORD -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -KinesisEgressRecord = _reflection.GeneratedProtocolMessageType('KinesisEgressRecord', (_message.Message,), { - 'DESCRIPTOR' : _KINESISEGRESSRECORD, - '__module__' : 'kinesis_egress_pb2' - # @@protoc_insertion_point(class_scope:org.apache.flink.statefun.flink.io.KinesisEgressRecord) - }) -_sym_db.RegisterMessage(KinesisEgressRecord) - - -DESCRIPTOR._options = None -# @@protoc_insertion_point(module_scope) diff --git a/statefun-flink/statefun-flink-io/src/main/protobuf/kafka-egress.proto b/statefun-sdk-protos/src/main/protobuf/io/kafka-egress.proto similarity index 100% rename from statefun-flink/statefun-flink-io/src/main/protobuf/kafka-egress.proto rename to statefun-sdk-protos/src/main/protobuf/io/kafka-egress.proto diff --git a/statefun-flink/statefun-flink-io/src/main/protobuf/kinesis-egress.proto b/statefun-sdk-protos/src/main/protobuf/io/kinesis-egress.proto similarity index 100% rename from statefun-flink/statefun-flink-io/src/main/protobuf/kinesis-egress.proto rename to statefun-sdk-protos/src/main/protobuf/io/kinesis-egress.proto diff --git a/statefun-sdk-protos/src/main/protobuf/request-reply.proto b/statefun-sdk-protos/src/main/protobuf/sdk/request-reply.proto similarity index 100% rename from statefun-sdk-protos/src/main/protobuf/request-reply.proto rename to statefun-sdk-protos/src/main/protobuf/sdk/request-reply.proto
