This is an automated email from the ASF dual-hosted git repository.
kamir pushed a change to branch kamir-patch-1
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
from 1632e748 Update README.md
add a98fdba8 Prepared the JavaKafkaTopicSource incl. Test.
add a4af308f Prepared the JavaKafkaTopicSource incl. Test.
add 01164f3c draft PR for checking the visibility issue in method: def
readKafkaTopic(topicName: String)
add a1022117 step 1 for JavaKafkaSource is done.
add cd4936e6 added the mapping for KafkaTopicSource to JavaKafkaTopicSource
add f46fd007 Finalized the KafkaSource
add cd7b6ea9 added KafkaTopicSink component
add 78e25d65 finished the KafkaSink implementation for Java Platform
add c8af60b8 added SparkKafkaTopicSource/Sink
add fd132887 fixed version problem in dependencies
add 31d7aa8d clean
add 4c8282b5 fixing the serialization problems
add d05730f6 fixed test issue
new d65f7cca merged kafka source feature into kamir-patch-1-branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.gitignore | 9 +
.move.sh | 69 ++++++
.../WayangApiJsonException.scala => build/_env.sh | 14 +-
.../scala/org/apache/wayang/api/DataQuanta.scala | 38 +++
.../org/apache/wayang/api/DataQuantaBuilder.scala | 15 ++
.../org/apache/wayang/api/JavaPlanBuilder.scala | 11 +-
wayang-commons/wayang-basic/pom.xml | 5 +
.../wayang/basic/operators/KafkaTopicSink.java | 230 ++++++++++++++++++
.../wayang/basic/operators/KafkaTopicSource.java | 268 +++++++++++++++++++++
.../org/apache/wayang/core/api/Configuration.java | 3 +-
.../wayang/core/function/FunctionDescriptor.java | 4 +-
.../core/function/TransformationDescriptor.java | 11 +-
.../optimizer/ProbabilisticDoubleInterval.java | 11 +-
.../optimizer/ProbabilisticIntervalEstimate.java | 13 +-
.../optimizer/cardinality/CardinalityEstimate.java | 6 +-
.../cardinality/CardinalityEstimator.java | 4 +-
.../cardinality/DefaultCardinalityEstimator.java | 13 +-
.../wayang/core/optimizer/costs/LoadEstimate.java | 8 +-
.../wayang/core/optimizer/costs/LoadProfile.java | 3 +-
.../wayang/core/plan/wayangplan/InputSlot.java | 7 +-
.../core/plan/wayangplan/OperatorAlternative.java | 9 +-
.../wayang/core/plan/wayangplan/OperatorBase.java | 9 +-
.../core/plan/wayangplan/OperatorContainer.java | 3 +-
.../core/plan/wayangplan/OperatorContainers.java | 5 +-
.../apache/wayang/core/plan/wayangplan/Slot.java | 11 +-
.../wayang/core/plan/wayangplan/SlotMapping.java | 5 +-
.../wayang/core/plan/wayangplan/UnarySink.java | 6 +
.../wayang/core/types/BasicDataUnitType.java | 7 +-
.../org/apache/wayang/core/types/DataSetType.java | 7 +-
.../org/apache/wayang/core/types/DataUnitType.java | 5 +-
...che-Wayang-Architecture-Overview-MK-v01.graffle | Bin 0 -> 1621385 bytes
.../Apache Wayang meets Apache Kafka.docx | Bin 0 -> 676382 bytes
.../Scenario-cross-organizational-sharing.graffle | Bin 0 -> 153329 bytes
wayang-platforms/wayang-java/pom.xml | 1 +
.../java/mapping/KafkaTopicSinkMapping.java} | 24 +-
.../java/mapping/KafkaTopicSourceMapping.java} | 22 +-
.../org/apache/wayang/java/mapping/Mappings.java | 4 +-
...vaTextFileSink.java => JavaKafkaTopicSink.java} | 69 ++++--
.../java/operators/JavaKafkaTopicSource.java | 131 ++++++++++
.../main/resources/wayang-java-defaults.properties | 32 +++
.../resources/wayang-kafka-defaults.properties | 9 +-
.../java/operators/JavaKafkaTopicSourceTest.java | 141 +++++++++++
.../apache/wayang/java/test/KafkaClientTest.java | 134 +++++++++++
wayang-platforms/wayang-spark/pom.xml | 7 +
...eansMapping.java => KafkaTopicSinkMapping.java} | 20 +-
...nsMapping.java => KafkaTopicSourceMapping.java} | 17 +-
.../org/apache/wayang/spark/mapping/Mappings.java | 5 +-
...kTextFileSink.java => SparkKafkaTopicSink.java} | 53 +++-
...tFileSource.java => SparkKafkaTopicSource.java} | 37 ++-
49 files changed, 1380 insertions(+), 135 deletions(-)
create mode 100755 .move.sh
copy
wayang-api/wayang-api-json/src/main/scala/exception/WayangApiJsonException.scala
=> build/_env.sh (81%)
create mode 100644
wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/KafkaTopicSink.java
create mode 100644
wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/KafkaTopicSource.java
create mode 100644
wayang-docs/src/main/resources/assets/omnigraffle/Apache-Wayang-Architecture-Overview-MK-v01.graffle
create mode 100644 wayang-docs/src/main/resources/extending_wayang/Apache
Wayang meets Apache Kafka.docx
create mode 100644
wayang-docs/src/main/resources/extending_wayang/Scenario-cross-organizational-sharing.graffle
copy
wayang-platforms/{wayang-spark/src/main/java/org/apache/wayang/spark/mapping/ml/KMeansMapping.java
=>
wayang-java/src/main/java/org/apache/wayang/java/mapping/KafkaTopicSinkMapping.java}
(69%)
copy
wayang-platforms/{wayang-spark/src/main/java/org/apache/wayang/spark/mapping/ml/KMeansMapping.java
=>
wayang-java/src/main/java/org/apache/wayang/java/mapping/KafkaTopicSourceMapping.java}
(70%)
copy
wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/{JavaTextFileSink.java
=> JavaKafkaTopicSink.java} (58%)
create mode 100644
wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/JavaKafkaTopicSource.java
copy
wayang-api/wayang-api-python/src/main/resources/wayang-api-python-defaults.properties
=>
wayang-platforms/wayang-java/src/main/resources/wayang-kafka-defaults.properties
(55%)
create mode 100644
wayang-platforms/wayang-java/src/test/java/org/apache/wayang/java/operators/JavaKafkaTopicSourceTest.java
create mode 100644
wayang-platforms/wayang-java/src/test/java/org/apache/wayang/java/test/KafkaClientTest.java
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/{ml/KMeansMapping.java
=> KafkaTopicSinkMapping.java} (74%)
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/{ml/KMeansMapping.java
=> KafkaTopicSourceMapping.java} (77%)
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/{SparkTextFileSink.java
=> SparkKafkaTopicSink.java} (68%)
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/{SparkTextFileSource.java
=> SparkKafkaTopicSource.java} (75%)
