This is an automated email from the ASF dual-hosted git repository. aloalt pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/wayang.git
commit e6ce5a9337c2f4fed6d2730a85df203a57e4954f Merge: 07cc9fd2d a47315251 Author: Alexander Alten <[email protected]> AuthorDate: Wed Jan 7 09:51:55 2026 +0100 Merge pull request #646 from novatechflow/feature/spark-dataframes Feature/spark dataframes README.md | 14 ++ guides/spark-datasets.md | 61 ++++++++ .../scala/org/apache/wayang/api/DataQuanta.scala | 17 ++- .../org/apache/wayang/api/JavaPlanBuilder.scala | 13 +- .../scala/org/apache/wayang/api/PlanBuilder.scala | 6 +- .../apache/wayang/basic/operators/ParquetSink.java | 58 ++++++++ .../wayang/basic/operators/ParquetSource.java | 12 ++ .../wayang/spark/channels/ChannelConversions.java | 51 ++++++- .../wayang/spark/channels/DatasetChannel.java | 123 ++++++++++++++++ .../org/apache/wayang/spark/mapping/Mappings.java | 3 +- .../wayang/spark/mapping/ParquetSinkMapping.java | 56 ++++++++ .../spark/operators/SparkDatasetToRddOperator.java | 96 +++++++++++++ .../wayang/spark/operators/SparkParquetSink.java | 90 ++++++++++++ .../wayang/spark/operators/SparkParquetSource.java | 36 +++-- .../spark/operators/SparkRddToDatasetOperator.java | 104 ++++++++++++++ .../wayang/spark/util/DatasetConverters.java | 157 +++++++++++++++++++++ .../resources/wayang-spark-defaults.properties | 22 +++ .../wayang/spark/operators/DatasetChannelTest.java | 98 +++++++++++++ .../wayang/spark/operators/DatasetTestUtils.java | 84 +++++++++++ .../operators/SparkDatasetToRddOperatorTest.java | 46 ++++++ .../spark/operators/SparkOperatorTestBase.java | 47 ++++-- .../spark/operators/SparkParquetSinkTest.java | 70 +++++++++ .../SparkParquetSourceDatasetOutputTest.java | 57 ++++++++ .../operators/SparkRddToDatasetOperatorTest.java | 43 ++++++ .../apache/wayang/spark/test/ChannelFactory.java | 17 +++ 25 files changed, 1348 insertions(+), 33 deletions(-)
