This is an automated email from the ASF dual-hosted git repository.
aloalt pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/wayang.git
from 07cc9fd2d Merge pull request #651 from aleudework/modified-rest-server
add fabc62862 Spark DataFrames support / Optimizer load profiles
add 5f42f3c52 Update readme / add documentation
add 599508d25 add license header
add 7b5d3b191 Add Dataset flag to read/write Parquet APIs and update docs
to reflect the unified interface.
add a738c3fb6 Unify parquet defaults in JavaPlanBuilder [INFO] BUILD
SUCCESS, this solves the CR
add fbc4da06d Add Java overloads for readParquet [INFO] BUILD SUCCESS
add 0af3ec8a9 Import JvmOverloads for readParquet [INFO] BUILD SUCCESS
add 67f5dcc12 Restore Java parquet overload [INFO] BUILD SUCCESS
add a47315251 Fix parquet overload defaults [INFO] BUILD SUCCESS
new e6ce5a933 Merge pull request #646 from
novatechflow/feature/spark-dataframes
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 14 ++
guides/spark-datasets.md | 61 ++++++++
.../scala/org/apache/wayang/api/DataQuanta.scala | 17 ++-
.../org/apache/wayang/api/JavaPlanBuilder.scala | 13 +-
.../scala/org/apache/wayang/api/PlanBuilder.scala | 6 +-
.../apache/wayang/basic/operators/ParquetSink.java | 58 ++++++++
.../wayang/basic/operators/ParquetSource.java | 12 ++
.../wayang/spark/channels/ChannelConversions.java | 51 ++++++-
.../{RddChannel.java => DatasetChannel.java} | 93 +++++-------
.../org/apache/wayang/spark/mapping/Mappings.java | 3 +-
.../{SortMapping.java => ParquetSinkMapping.java} | 18 +--
...perator.java => SparkDatasetToRddOperator.java} | 66 ++++-----
.../wayang/spark/operators/SparkParquetSink.java | 90 ++++++++++++
.../wayang/spark/operators/SparkParquetSource.java | 36 +++--
...ileSink.java => SparkRddToDatasetOperator.java} | 66 ++++-----
.../wayang/spark/util/DatasetConverters.java | 157 +++++++++++++++++++++
.../resources/wayang-spark-defaults.properties | 22 +++
.../wayang/spark/operators/DatasetChannelTest.java | 98 +++++++++++++
.../wayang/spark/operators/DatasetTestUtils.java | 84 +++++++++++
...est.java => SparkDatasetToRddOperatorTest.java} | 39 ++---
.../spark/operators/SparkOperatorTestBase.java | 47 ++++--
.../spark/operators/SparkParquetSinkTest.java | 70 +++++++++
.../SparkParquetSourceDatasetOutputTest.java | 57 ++++++++
.../operators/SparkRddToDatasetOperatorTest.java | 44 +++---
.../apache/wayang/spark/test/ChannelFactory.java | 17 +++
25 files changed, 1022 insertions(+), 217 deletions(-)
create mode 100644 guides/spark-datasets.md
create mode 100644
wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSink.java
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/{RddChannel.java
=> DatasetChannel.java} (53%)
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/{SortMapping.java
=> ParquetSinkMapping.java} (74%)
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/{SparkCacheOperator.java
=> SparkDatasetToRddOperator.java} (54%)
create mode 100644
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/SparkParquetSink.java
copy
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/{SparkTextFileSink.java
=> SparkRddToDatasetOperator.java} (52%)
create mode 100644
wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/util/DatasetConverters.java
create mode 100644
wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/DatasetChannelTest.java
create mode 100644
wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/DatasetTestUtils.java
copy
wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/{SparkCollectionSourceTest.java
=> SparkDatasetToRddOperatorTest.java} (52%)
create mode 100644
wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkParquetSinkTest.java
create mode 100644
wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkParquetSourceDatasetOutputTest.java
copy
wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/util/LimitedInputStreamTest.java
=>
wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkRddToDatasetOperatorTest.java
(51%)
