This is an automated email from the ASF dual-hosted git repository.
aloalt pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
from 88dce06c Merge pull request #462 from apache/2pk03-patch-2
add a98fdba8 Prepared the JavaKafkaTopicSource incl. Test.
add a4af308f Prepared the JavaKafkaTopicSource incl. Test.
add 01164f3c draft PR for checking the visibility issue in method: def
readKafkaTopic(topicName: String)
add a1022117 step 1 for JavaKafkaSource is done.
add cd4936e6 added the mapping for KafkaTopicSource to JavaKafkaTopicSource
add f46fd007 Finalized the KafkaSource
add cd7b6ea9 added KafkaTopicSink component
add 78e25d65 finished the KafkaSink implementation for Java Platform
add c8af60b8 added SparkKafkaTopicSource/Sink
add fd132887 fixed version problem in dependencies
add 31d7aa8d clean
add 4c8282b5 fixing the serialization problems
add d05730f6 fixed test issue
add d65f7cca merged kafka source feature into kamir-patch-1-branch
add 9526794c clean patch
add 94b7ef63 updated dependencies
add 24487667 Merge branch 'main' into kamir-patch-1
add db678d77 Update DefaultCardinalityEstimator.java
add 9e37924d Update DefaultCardinalityEstimator.java
add f3293232 Update DefaultCardinalityEstimator.java
add 0e5fafe6 Merge branch 'kamir-patch-1' of
https://github.com/apache/incubator-wayang into kamir-patch-1
add 31cdbffd deactivated non relevant tests - KafkaTopicSource,
JavaKafkaTopicSource
add eff302f1 Update backend.yml
add 2d56229f Merge branch 'kamir-patch-1' of
https://github.com/apache/incubator-wayang into kamir-patch-1
add 26fc5de1 fixes jackson-databind version (back to 2.10.0)
add b998d5d7 Update backend.yml
add 7895720f Merge pull request #464 from apache/kamir-patch-1
No new revisions were added by this update.
Summary of changes:
.github/workflows/backend.yml | 2 +-
.gitignore | 10 +
.../WayangApiJsonException.scala => build/_env.sh | 13 +-
.../scala/org/apache/wayang/api/DataQuanta.scala | 38 +++
.../org/apache/wayang/api/DataQuantaBuilder.scala | 15 ++
.../org/apache/wayang/api/JavaPlanBuilder.scala | 13 +-
wayang-commons/wayang-basic/pom.xml | 5 +
.../wayang/basic/operators/KafkaTopicSink.java | 230 +++++++++++++++++
.../wayang/basic/operators/KafkaTopicSource.java | 286 +++++++++++++++++++++
.../org/apache/wayang/core/api/Configuration.java | 3 +-
.../wayang/core/function/FunctionDescriptor.java | 4 +-
.../core/function/TransformationDescriptor.java | 11 +-
.../optimizer/ProbabilisticDoubleInterval.java | 11 +-
.../optimizer/ProbabilisticIntervalEstimate.java | 13 +-
.../optimizer/cardinality/CardinalityEstimate.java | 6 +-
.../cardinality/CardinalityEstimator.java | 4 +-
.../cardinality/DefaultCardinalityEstimator.java | 16 +-
.../wayang/core/optimizer/costs/LoadEstimate.java | 8 +-
.../wayang/core/optimizer/costs/LoadProfile.java | 3 +-
.../wayang/core/plan/wayangplan/InputSlot.java | 7 +-
.../core/plan/wayangplan/OperatorAlternative.java | 9 +-
.../wayang/core/plan/wayangplan/OperatorBase.java | 9 +-
.../core/plan/wayangplan/OperatorContainer.java | 3 +-
.../core/plan/wayangplan/OperatorContainers.java | 5 +-
.../apache/wayang/core/plan/wayangplan/Slot.java | 11 +-
.../wayang/core/plan/wayangplan/SlotMapping.java | 5 +-
.../wayang/core/plan/wayangplan/UnarySink.java | 6 +
.../wayang/core/types/BasicDataUnitType.java | 7 +-
.../org/apache/wayang/core/types/DataSetType.java | 7 +-
.../org/apache/wayang/core/types/DataUnitType.java | 5 +-
...che-Wayang-Architecture-Overview-MK-v01.graffle | Bin 0 -> 1621385 bytes
.../Apache Wayang meets Apache Kafka.docx | Bin 0 -> 676382 bytes
.../Scenario-cross-organizational-sharing.graffle | Bin 0 -> 153329 bytes
wayang-platforms/wayang-java/pom.xml | 17 ++
.../java/mapping/KafkaTopicSinkMapping.java} | 24 +-
.../java/mapping/KafkaTopicSourceMapping.java} | 22 +-
.../org/apache/wayang/java/mapping/Mappings.java | 4 +-
...vaTextFileSink.java => JavaKafkaTopicSink.java} | 69 +++--
.../java/operators/JavaKafkaTopicSource.java | 131 ++++++++++
.../main/resources/wayang-java-defaults.properties | 32 +++
.../resources/wayang-kafka-defaults.properties | 11 +-
.../java/operators/JavaKafkaTopicSourceTest.java | 143 +++++++++++
.../apache/wayang/java/test/KafkaClientTest.java | 163 ++++++++++++
wayang-platforms/wayang-spark/pom.xml | 8 +
...eansMapping.java => KafkaTopicSinkMapping.java} | 20 +-
...nsMapping.java => KafkaTopicSourceMapping.java} | 17 +-
.../org/apache/wayang/spark/mapping/Mappings.java | 5 +-
...kTextFileSink.java => SparkKafkaTopicSink.java} | 53 +++-
...tFileSource.java => SparkKafkaTopicSource.java} | 39 ++-
49 files changed, 1382 insertions(+), 141 deletions(-)
copy
wayang-api/wayang-api-json/src/main/scala/exception/WayangApiJsonException.scala
=> build/_env.sh (81%)
create mode 100644
wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/KafkaTopicSink.java
create mode 100644
wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/KafkaTopicSource.java
create mode 100644
wayang-docs/src/main/resources/assets/omnigraffle/Apache-Wayang-Architecture-Overview-MK-v01.graffle
create mode 100644 wayang-docs/src/main/resources/extending_wayang/Apache
Wayang meets Apache Kafka.docx
create mode 100644
wayang-docs/src/main/resources/extending_wayang/Scenario-cross-organizational-sharing.graffle
copy
wayang-platforms/{wayang-spark/src/main/java/org/apache/wayang/spark/mapping/ml/KMeansMapping.java
=>
wayang-java/src/main/java/org/apache/wayang/java/mapping/KafkaTopicSinkMapping.java}
(69%)
copy
wayang-platforms/{wayang-spark/src/main/java/org/apache/wayang/spark/mapping/ml/KMeansMapping.java
=>
wayang-java/src/main/java/org/apache/wayang/java/mapping/KafkaTopicSourceMapping.java}
(70%)
copy
wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/{JavaTextFileSink.java
=> JavaKafkaTopicSink.java} (58%)
create mode 100644
wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/JavaKafkaTopicSource.java
copy python/old_code/pywayang/src/pywy/config/__init__.py =>
wayang-platforms/wayang-java/src/main/resources/wayang-kafka-defaults.properties
(55%)
create mode 100644
wayang-platforms/wayang-java/src/test/java/org/apache/wayang/java/operators/JavaKafkaTopicSourceTest.java
create mode 100644
wayang-platforms/wayang-java/src/test/java/org/apache/wayang/java/test/KafkaClientTest.java
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/{ml/KMeansMapping.java
=> KafkaTopicSinkMapping.java} (74%)
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/{ml/KMeansMapping.java
=> KafkaTopicSourceMapping.java} (77%)
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/{SparkTextFileSink.java
=> SparkKafkaTopicSink.java} (68%)
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/{SparkTextFileSource.java
=> SparkKafkaTopicSource.java} (75%)
