This is an automated email from the ASF dual-hosted git repository. bertty pushed a commit to branch WAYANG-8 in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 828c9c235dd7376b7dd937bc1c70baec79eab47b Merge: ff0cb9f 24a55f8 Author: berttty <[email protected]> AuthorDate: Fri Mar 5 13:30:01 2021 -0300 Merge pull request #9 from ro-pardo/python-api Python api .travis.yml | 35 +++- README.md | 6 +- pom.xml | 47 ++---- .../pyplangenerator.sh | 4 +- .../Action.java => protobuf/pywayangplan.proto | 71 +++++--- .../config/__init__.py | 4 +- pywayang/config/config_reader.py | 51 ++++++ .../config/pywayang_config.ini | 22 ++- .../graph/__init__.py | 4 +- pywayang/graph/graph.py | 68 ++++++++ pywayang/graph/node.py | 44 +++++ pywayang/graph/traversal.py | 49 ++++++ pywayang/graph/visitant.py | 51 ++++++ .../orchestrator/__init__.py | 5 +- pywayang/orchestrator/dataquanta.py | 136 +++++++++++++++ pywayang/orchestrator/main.py | 76 +++++++++ pywayang/orchestrator/operator.py | 110 ++++++++++++ .../orchestrator/plan.py | 22 ++- .../protobuf/__init__.py | 2 +- pywayang/protobuf/planwriter.py | 152 +++++++++++++++++ wayang-api/pom.xml | 1 + wayang-api/wayang-api-python/pom.xml | 46 ++++- .../wayang/api/python/executor/ProcessFeeder.java | 177 ++++++++++++++++++++ .../api/python/executor/ProcessReceiver.java | 52 ++++++ .../api/python/executor/PythonProcessCaller.java | 118 +++++++++++++ .../api/python/executor/PythonWorkerManager.java | 64 +++++++ .../wayang/api/python/executor/ReaderIterator.java | 86 ++++++++++ .../wayang/api/python/function/PythonUdf.java | 19 +-- .../api/python/function/WrappedPythonFunction.java | 43 +++++ .../wayang-api-python/src/main/python/worker.py | 186 +++++++++++++++++++++ .../wayang-api-python-defaults.properties | 3 +- .../org/apache/wayang/api/python/BasicPython.java | 67 ++++++++ .../org/apache/wayang/api/python/SortPython.java | 102 +++++++++++ wayang-api/wayang-api-rest/pom.xml | 97 +++++++++++ .../org/apache/wayang/api/rest/WayangRESTAPI.java | 20 +-- .../api/rest/server/spring/WayangApplication.java | 22 +-- .../rest/server/spring/decoder/pyplanreader.sh} | 4 +- .../server/spring/general/WayangController.java | 159 ++++++++++++++++++ .../src/main/resources/application.properties} | 2 +- wayang-commons/pom.xml | 13 +- .../wayang/basic/operators/SampleOperator.java | 6 +- .../wayang/basic/operators/TextFileSource.java | 6 +- .../wayang/basic/operators/TextFileSourceTest.java | 7 +- wayang-commons/wayang-core/pom.xml | 8 + .../org/apache/wayang/core/api/Configuration.java | 6 +- .../main/java/org/apache/wayang/core/api/Job.java | 6 +- .../org/apache/wayang/core/api/WayangContext.java | 6 +- .../core/api/configuration/CollectionProvider.java | 6 +- .../configuration/ExplicitCollectionProvider.java | 6 +- .../core/api/configuration/KeyValueProvider.java | 6 +- .../core/api/configuration/ValueProvider.java | 6 +- .../wayang/core/mapping/PlanTransformation.java | 6 +- .../wayang/core/optimizer/OptimizationContext.java | 6 +- .../wayang/core/optimizer/SanityChecker.java | 6 +- .../cardinality/CardinalityEstimatorManager.java | 6 +- .../optimizer/cardinality/CardinalityPusher.java | 6 +- .../SwitchForwardCardinalityEstimator.java | 4 +- .../optimizer/channels/ChannelConversionGraph.java | 6 +- .../optimizer/costs/LoadProfileEstimators.java | 6 +- .../optimizer/enumeration/ExecutionTaskFlow.java | 6 +- .../enumeration/ExecutionTaskFlowCompiler.java | 4 +- .../enumeration/LatentOperatorPruningStrategy.java | 10 +- .../core/optimizer/enumeration/LoopEnumerator.java | 4 +- .../optimizer/enumeration/PlanEnumeration.java | 6 +- .../core/optimizer/enumeration/PlanEnumerator.java | 6 +- .../optimizer/enumeration/PlanImplementation.java | 6 +- .../enumeration/StageAssignmentTraversal.java | 6 +- .../wayang/core/plan/executionplan/Channel.java | 8 +- .../core/plan/executionplan/ExecutionPlan.java | 6 +- .../core/plan/wayangplan/ExecutionOperator.java | 4 +- .../wayang/core/plan/wayangplan/LoopIsolator.java | 4 +- .../core/plan/wayangplan/OperatorContainer.java | 4 +- .../wayang/core/plan/wayangplan/PlanTraversal.java | 6 +- .../wayang/core/plan/wayangplan/SlotMapping.java | 6 +- .../wayang/core/plan/wayangplan/WayangPlan.java | 6 +- .../traversal/AbstractTopologicalTraversal.java | 6 +- .../core/platform/AbstractChannelInstance.java | 4 +- .../core/platform/CardinalityBreakpoint.java | 6 +- .../core/platform/CrossPlatformExecutor.java | 6 +- .../wayang/core/platform/ExecutorTemplate.java | 6 +- .../org/apache/wayang/core/platform/Junction.java | 6 +- .../platform/lineage/LazyExecutionLineageNode.java | 4 +- .../core/profiling/CardinalityRepository.java | 6 +- .../apache/wayang/core/profiling/ExecutionLog.java | 6 +- .../core/util/AbstractReferenceCountable.java | 7 +- .../java/org/apache/wayang/core/util/Action.java | 4 +- .../org/apache/wayang/core/util/JuelUtils.java | 6 +- .../java/org/apache/wayang/core/util/Logging.java | 6 +- .../apache/wayang/core/util/ReflectionUtils.java | 6 +- .../apache/wayang/core/util/fs/FileSystems.java | 8 +- .../wayang/core/util/fs/HadoopFileSystem.java | 10 +- .../wayang/core/util/fs/LocalFileSystem.java | 6 +- .../core/util/ConsumerIteratorAdapterTest.java | 6 +- wayang-commons/wayang-serializable/pom.xml | 80 +++++++++ .../src/main/proto/pywayangplan.proto | 73 ++++++++ .../flink/compiler/WayangFileOutputFormat.java | 6 +- .../wayang/flink/platform/FlinkPlatform.java | 6 +- .../wayang/giraph/Algorithm/PageRankAlgorithm.java | 7 +- .../giraph/operators/GiraphPageRankOperator.java | 6 +- .../operators/GraphChiPageRankOperator.java | 6 +- .../wayang/java/operators/JavaObjectFileSink.java | 4 +- .../java/operators/JavaObjectFileSource.java | 4 +- wayang-platforms/wayang-jdbc-template/pom.xml | 17 ++ .../apache/wayang/jdbc/execution/JdbcExecutor.java | 6 +- .../wayang/jdbc/operators/JdbcTableSource.java | 4 +- .../wayang/jdbc/operators/SqlToStreamOperator.java | 4 +- .../spark/operators/SparkObjectFileSink.java | 4 +- .../spark/operators/SparkObjectFileSource.java | 6 +- .../wayang/spark/platform/SparkPlatform.java | 6 +- .../wayang/profiler/java/OperatorProfiler.java | 6 +- .../wayang/profiler/java/SourceProfiler.java | 5 +- .../profiler/log/DynamicLoadProfileEstimators.java | 6 +- .../wayang/profiler/log/GeneticOptimizerApp.java | 6 +- .../apache/wayang/profiler/log/LogEvaluator.java | 6 +- .../profiler/spark/SparkOperatorProfiler.java | 6 +- .../wayang/profiler/util/ProfilingUtils.java | 4 +- .../apache/wayang/profiler/util/RrdAccessor.java | 6 +- {wayang-api => wayang-resources}/pom.xml | 39 +++-- .../src/main/assembly/resources.xml | 39 ++--- .../src/main/resources/assemblies/resources.xml | 39 ++--- .../apache/wayang/tests/GraphChiIntegrationIT.java | 12 +- 121 files changed, 2557 insertions(+), 412 deletions(-)
