This is an automated email from the ASF dual-hosted git repository. bertty pushed a commit to branch python-platform in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 2205368b051289e896c2f7768aac9906f3d3939c Author: Bertty Contreras-Rojas <[email protected]> AuthorDate: Wed Apr 6 13:17:05 2022 +0200 [WAYANG-#8] rename to pywy and organization of class Signed-off-by: bertty <[email protected]> --- python/resources/test.input | 4 ++++ python/src/pywayang/__main__.py | 1 - .../python/operators/PythonExecutionOperator.py | 7 ------ .../platforms/python/operators/__init__.py | 11 --------- .../pywayang/platforms/python/plugin/__init__.py | 0 python/src/{pywayang => pywy}/__init__.py | 2 +- .../src/{pywayang => pywy}/config/config_reader.py | 0 .../{pywayang => pywy}/config/pywayang_config.ini | 0 python/src/{pywayang => pywy}/context.py | 6 ++--- python/src/{pywayang => pywy}/dataquanta.py | 8 ++----- python/src/{pywayang => pywy}/graph/__init__.py | 0 python/src/{pywayang => pywy}/graph/graph.py | 2 +- python/src/{pywayang => pywy}/graph/node.py | 0 python/src/{pywayang => pywy}/graph/traversal.py | 2 +- python/src/{pywayang => pywy}/graph/visitant.py | 0 .../{pywayang => pywy}/orchestrator/__init__.py | 0 .../{pywayang => pywy}/orchestrator/dataquanta.py | 8 +++---- .../orchestrator/execdirectly.py | 4 ++-- python/src/{pywayang => pywy}/orchestrator/main.py | 4 ++-- .../{pywayang => pywy}/orchestrator/operator.py | 6 ++--- python/src/{pywayang => pywy}/orchestrator/plan.py | 0 .../operator => pywy/platforms}/__init__.py | 0 .../platforms => pywy/platforms/basic}/__init__.py | 0 python/src/pywy/platforms/basic/channel.py | 20 +++++++++++++++++ .../platforms/basic/mapping.py} | 15 +++---------- .../{pywayang => pywy/platforms/basic}/platform.py | 0 .../{pywayang => pywy/platforms/basic}/plugin.py | 14 +++++++----- .../platforms/python/__init__.py | 0 .../platforms/python/channels.py | 23 +------------------ .../platforms/python/compiler/__init__.py | 0 .../platforms/python/execution/__init__.py | 0 python/src/pywy/platforms/python/mappings.py | 10 +++++++++ .../platforms/python/operators/PyFilterOperator.py | 10 ++++----- .../python/operators/PyTextFileSinkOperator.py | 6 ++--- .../python/operators/PyTextFileSourceOperator.py | 6 ++--- .../python/operators/PythonExecutionOperator.py | 7 ++++++ .../pywy/platforms/python/operators/__init__.py | 11 +++++++++ .../src/pywy/platforms/python/platform/__init__.py | 1 + .../src/pywy/platforms/python/platform/platform.py | 6 +++++ .../src/pywy/platforms/python/plugin/__init__.py | 1 + python/src/pywy/platforms/python/plugin/plugin.py | 10 +++++++++ python/src/pywy/plugins.py | 10 +++++++++ python/src/{pywayang => pywy}/test.py | 26 +++++++++++----------- .../{pywayang/protobuf => pywy/test}/__init__.py | 0 python/src/{pywayang => pywy}/test/demo_testing.py | 0 .../src/{pywayang => pywy}/test/full_java_test.py | 4 ++-- .../src/{pywayang => pywy}/test/full_spark_test.py | 4 ++-- .../python/platform => pywy/translate}/__init__.py | 0 .../test => pywy/translate/protobuf}/__init__.py | 0 .../translate}/protobuf/planwriter.py | 3 +-- python/src/pywy/translate/translator.py | 12 ++++++++++ python/src/{pywayang => pywy}/types.py | 0 python/src/pywy/wayangplan/__init__.py | 15 +++++++++++++ .../{pywayang/operator => pywy/wayangplan}/base.py | 2 +- .../{pywayang/operator => pywy/wayangplan}/sink.py | 15 +++++++++++-- .../operator => pywy/wayangplan}/source.py | 2 +- .../operator => pywy/wayangplan}/unary.py | 4 ++-- python/src/pywy/wayangplan/wayang.py | 11 +++++++++ 58 files changed, 195 insertions(+), 118 deletions(-) diff --git a/python/resources/test.input b/python/resources/test.input new file mode 100644 index 00000000..61a99ae2 --- /dev/null +++ b/python/resources/test.input @@ -0,0 +1,4 @@ +la +lala +a +lalala \ No newline at end of file diff --git a/python/src/pywayang/__main__.py b/python/src/pywayang/__main__.py deleted file mode 100644 index 0acdb618..00000000 --- a/python/src/pywayang/__main__.py +++ /dev/null @@ -1 +0,0 @@ -print("hello"); \ No newline at end of file diff --git a/python/src/pywayang/platforms/python/operators/PythonExecutionOperator.py b/python/src/pywayang/platforms/python/operators/PythonExecutionOperator.py deleted file mode 100644 index 4a79616c..00000000 --- a/python/src/pywayang/platforms/python/operators/PythonExecutionOperator.py +++ /dev/null @@ -1,7 +0,0 @@ -from pywayang.operator.base import WyOperator -from pywayang.platforms.python.channels import Channel - -class PythonExecutionOperator(WyOperator): - - def execute(self, inputs: Channel, output: Channel): - pass \ No newline at end of file diff --git a/python/src/pywayang/platforms/python/operators/__init__.py b/python/src/pywayang/platforms/python/operators/__init__.py deleted file mode 100644 index 7a555422..00000000 --- a/python/src/pywayang/platforms/python/operators/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from pywayang.platforms.python.operators.PythonExecutionOperator import PythonExecutionOperator -from pywayang.platforms.python.operators.PyFilterOperator import PyFilterOperator -from pywayang.platforms.python.operators.PyTextFileSourceOperator import PyTextFileSourceOperator -from pywayang.platforms.python.operators.PyTextFileSinkOperator import PyTextFileSinkOperator - -__ALL__ = [ - PythonExecutionOperator, - PyFilterOperator, - PyTextFileSourceOperator, - PyTextFileSinkOperator -] \ No newline at end of file diff --git a/python/src/pywayang/platforms/python/plugin/__init__.py b/python/src/pywayang/platforms/python/plugin/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/python/src/pywayang/__init__.py b/python/src/pywy/__init__.py similarity index 93% rename from python/src/pywayang/__init__.py rename to python/src/pywy/__init__.py index 7f555045..38c001b3 100644 --- a/python/src/pywayang/__init__.py +++ b/python/src/pywy/__init__.py @@ -17,6 +17,6 @@ from .config import * from .orchestrator import * -from .protobuf import * +from pywy.translate.protobuf import * from .graph import * from .test import * \ No newline at end of file diff --git a/python/src/pywayang/config/config_reader.py b/python/src/pywy/config/config_reader.py similarity index 100% rename from python/src/pywayang/config/config_reader.py rename to python/src/pywy/config/config_reader.py diff --git a/python/src/pywayang/config/pywayang_config.ini b/python/src/pywy/config/pywayang_config.ini similarity index 100% rename from python/src/pywayang/config/pywayang_config.ini rename to python/src/pywy/config/pywayang_config.ini diff --git a/python/src/pywayang/context.py b/python/src/pywy/context.py similarity index 82% rename from python/src/pywayang/context.py rename to python/src/pywy/context.py index 95942ef7..1f2e883a 100644 --- a/python/src/pywayang/context.py +++ b/python/src/pywy/context.py @@ -1,6 +1,6 @@ -from pywayang.plugin import Plugin -from pywayang.dataquanta import DataQuanta -from pywayang.operator.source import TextFileSource +from pywy.platforms.basic.plugin import Plugin +from pywy.dataquanta import DataQuanta +from pywy.wayangplan.source import TextFileSource class WayangContext: """ diff --git a/python/src/pywayang/dataquanta.py b/python/src/pywy/dataquanta.py similarity index 77% rename from python/src/pywayang/dataquanta.py rename to python/src/pywy/dataquanta.py index 9cd892eb..c4e7b15c 100644 --- a/python/src/pywayang/dataquanta.py +++ b/python/src/pywy/dataquanta.py @@ -1,8 +1,5 @@ -from pywayang.types import (GenericTco, Predicate, Function, FlatmapFunction, IterableO) -from pywayang.operator.base import (WyOperator) -from pywayang.operator.unary import (FilterOperator, MapOperator, FlatmapOperator) -from pywayang.operator.sink import TextFileSink - +from pywy.types import (GenericTco, Predicate, Function, FlatmapFunction, IterableO) +from pywy.wayangplan import * class DataQuanta(GenericTco): """ @@ -13,7 +10,6 @@ class DataQuanta(GenericTco): def __init__(self, operator: WyOperator): self.operator = operator - def filter(self: "DataQuanta[T]", p: Predicate) -> "DataQuanta[T]" : return DataQuanta(FilterOperator(p)) diff --git a/python/src/pywayang/graph/__init__.py b/python/src/pywy/graph/__init__.py similarity index 100% rename from python/src/pywayang/graph/__init__.py rename to python/src/pywy/graph/__init__.py diff --git a/python/src/pywayang/graph/graph.py b/python/src/pywy/graph/graph.py similarity index 98% rename from python/src/pywayang/graph/graph.py rename to python/src/pywy/graph/graph.py index df5d2ae8..a66787fa 100644 --- a/python/src/pywayang/graph/graph.py +++ b/python/src/pywy/graph/graph.py @@ -15,7 +15,7 @@ # limitations under the License. # -from pywayang.graph.node import Node +from pywy.graph.node import Node import logging diff --git a/python/src/pywayang/graph/node.py b/python/src/pywy/graph/node.py similarity index 100% rename from python/src/pywayang/graph/node.py rename to python/src/pywy/graph/node.py diff --git a/python/src/pywayang/graph/traversal.py b/python/src/pywy/graph/traversal.py similarity index 97% rename from python/src/pywayang/graph/traversal.py rename to python/src/pywy/graph/traversal.py index 780dd821..63542a4e 100644 --- a/python/src/pywayang/graph/traversal.py +++ b/python/src/pywy/graph/traversal.py @@ -15,7 +15,7 @@ # limitations under the License. # -from pywayang.graph.visitant import Visitant +from pywy.graph.visitant import Visitant import logging diff --git a/python/src/pywayang/graph/visitant.py b/python/src/pywy/graph/visitant.py similarity index 100% rename from python/src/pywayang/graph/visitant.py rename to python/src/pywy/graph/visitant.py diff --git a/python/src/pywayang/orchestrator/__init__.py b/python/src/pywy/orchestrator/__init__.py similarity index 100% rename from python/src/pywayang/orchestrator/__init__.py rename to python/src/pywy/orchestrator/__init__.py diff --git a/python/src/pywayang/orchestrator/dataquanta.py b/python/src/pywy/orchestrator/dataquanta.py similarity index 98% rename from python/src/pywayang/orchestrator/dataquanta.py rename to python/src/pywy/orchestrator/dataquanta.py index e5495c91..4e5a5661 100644 --- a/python/src/pywayang/orchestrator/dataquanta.py +++ b/python/src/pywy/orchestrator/dataquanta.py @@ -15,10 +15,10 @@ # limitations under the License. # -from pywayang.orchestrator.operator import Operator -from pywayang.graph.graph import Graph -from pywayang.graph.traversal import Traversal -from pywayang.protobuf.planwriter import MessageWriter +from pywy.orchestrator.operator import Operator +from pywy.graph.graph import Graph +from pywy.graph.traversal import Traversal +from pywy.translate.protobuf.planwriter import MessageWriter import itertools import collections import logging diff --git a/python/src/pywayang/orchestrator/execdirectly.py b/python/src/pywy/orchestrator/execdirectly.py similarity index 97% rename from python/src/pywayang/orchestrator/execdirectly.py rename to python/src/pywy/orchestrator/execdirectly.py index ac845a66..f27b0bb7 100644 --- a/python/src/pywayang/orchestrator/execdirectly.py +++ b/python/src/pywy/orchestrator/execdirectly.py @@ -15,8 +15,8 @@ # limitations under the License. # -from pywayang.orchestrator.plan import Descriptor -from pywayang.orchestrator.dataquanta import DataQuantaBuilder +from pywy.orchestrator.plan import Descriptor +from pywy.orchestrator.dataquanta import DataQuantaBuilder import datetime diff --git a/python/src/pywayang/orchestrator/main.py b/python/src/pywy/orchestrator/main.py similarity index 97% rename from python/src/pywayang/orchestrator/main.py rename to python/src/pywy/orchestrator/main.py index 8304e0d0..163e570e 100644 --- a/python/src/pywayang/orchestrator/main.py +++ b/python/src/pywy/orchestrator/main.py @@ -15,8 +15,8 @@ # limitations under the License. # -from pywayang.orchestrator.plan import Descriptor -from pywayang.orchestrator.dataquanta import DataQuantaBuilder +from pywy.orchestrator.plan import Descriptor +from pywy.orchestrator.dataquanta import DataQuantaBuilder import datetime diff --git a/python/src/pywayang/orchestrator/operator.py b/python/src/pywy/orchestrator/operator.py similarity index 95% rename from python/src/pywayang/orchestrator/operator.py rename to python/src/pywy/orchestrator/operator.py index 87087642..bd996d16 100644 --- a/python/src/pywayang/orchestrator/operator.py +++ b/python/src/pywy/orchestrator/operator.py @@ -17,9 +17,9 @@ import pickle import cloudpickle -from pywayang.config.config_reader import get_source_types -from pywayang.config.config_reader import get_sink_types -from pywayang.config.config_reader import get_boundary_types +from pywy.config.config_reader import get_source_types +from pywy.config.config_reader import get_sink_types +from pywy.config.config_reader import get_boundary_types import logging pickle_protocol = pickle.HIGHEST_PROTOCOL diff --git a/python/src/pywayang/orchestrator/plan.py b/python/src/pywy/orchestrator/plan.py similarity index 100% rename from python/src/pywayang/orchestrator/plan.py rename to python/src/pywy/orchestrator/plan.py diff --git a/python/src/pywayang/operator/__init__.py b/python/src/pywy/platforms/__init__.py similarity index 100% rename from python/src/pywayang/operator/__init__.py rename to python/src/pywy/platforms/__init__.py diff --git a/python/src/pywayang/platforms/__init__.py b/python/src/pywy/platforms/basic/__init__.py similarity index 100% rename from python/src/pywayang/platforms/__init__.py rename to python/src/pywy/platforms/basic/__init__.py diff --git a/python/src/pywy/platforms/basic/channel.py b/python/src/pywy/platforms/basic/channel.py new file mode 100644 index 00000000..18d5c3d3 --- /dev/null +++ b/python/src/pywy/platforms/basic/channel.py @@ -0,0 +1,20 @@ +class Channel: + + def __init__(self): + pass + + def getchannel(self) -> 'Channel': + return self + + def gettype(self): + return type(self) + +class ChannelDescriptor: + + def __init__(self, channelType: type, isReusable: bool, isSuitableForBreakpoint: bool): + self.channelType = channelType + self.isReusable = isReusable + self.isSuitableForBreakpoint = isSuitableForBreakpoint + + def create_instance(self) -> Channel: + return self.channelType() \ No newline at end of file diff --git a/python/src/pywayang/platforms/python/mappings.py b/python/src/pywy/platforms/basic/mapping.py similarity index 60% rename from python/src/pywayang/platforms/python/mappings.py rename to python/src/pywy/platforms/basic/mapping.py index 7060ba07..63c7e4ca 100644 --- a/python/src/pywayang/platforms/python/mappings.py +++ b/python/src/pywy/platforms/basic/mapping.py @@ -1,7 +1,5 @@ from typing import Dict - -from pywayang.operator.base import WyOperator -from pywayang.platforms.python.operators import * +from pywy.wayangplan.base import WyOperator class Mapping: mappings: Dict[str, type] @@ -9,7 +7,7 @@ class Mapping: def __init__(self): self.mappings = {} - def add_mapping(self, operator: PythonExecutionOperator): + def add_mapping(self, operator: WyOperator): self.mappings[operator.name] = type(operator) def get_instanceof(self, operator: WyOperator): @@ -27,11 +25,4 @@ class Mapping: return str(self.mappings) def __repr__(self): - return self.__str__() - -OperatorMappings = Mapping() - -OperatorMappings.add_mapping(PyFilterOperator()) -OperatorMappings.add_mapping(PyTextFileSourceOperator()) -OperatorMappings.add_mapping(PyTextFileSinkOperator()) - + return self.__str__() \ No newline at end of file diff --git a/python/src/pywayang/platform.py b/python/src/pywy/platforms/basic/platform.py similarity index 100% rename from python/src/pywayang/platform.py rename to python/src/pywy/platforms/basic/platform.py diff --git a/python/src/pywayang/plugin.py b/python/src/pywy/platforms/basic/plugin.py similarity index 58% rename from python/src/pywayang/plugin.py rename to python/src/pywy/platforms/basic/plugin.py index 82d75431..9176e9f3 100644 --- a/python/src/pywayang/plugin.py +++ b/python/src/pywy/platforms/basic/plugin.py @@ -1,4 +1,5 @@ -from pywayang.platform import Platform +from pywy.platforms.basic.platform import Platform +from pywy.platforms.basic.mapping import Mapping class Plugin: """ @@ -10,9 +11,14 @@ class Plugin: """ platforms = [] + mappings: Mapping - def __init__(self, *platform:Platform): + def __init__(self, *platform:Platform, mappings: Mapping = Mapping()): self.platforms = list(platform) + self.mappings = mappings + + def get_mappings(self) -> Mapping: + return self.mappings def __str__(self): return "Platforms: {}".format(str(self.platforms)) @@ -21,7 +27,3 @@ class Plugin: return self.__str__() -# define the basic plugins that can be used -java = Plugin(Platform('java')) -spark = Plugin(Platform('spark')) -flink = Plugin(Platform('flink')) diff --git a/python/src/pywayang/platforms/python/__init__.py b/python/src/pywy/platforms/python/__init__.py similarity index 100% rename from python/src/pywayang/platforms/python/__init__.py rename to python/src/pywy/platforms/python/__init__.py diff --git a/python/src/pywayang/platforms/python/channels.py b/python/src/pywy/platforms/python/channels.py similarity index 72% rename from python/src/pywayang/platforms/python/channels.py rename to python/src/pywy/platforms/python/channels.py index a2863677..5f0b7a7a 100644 --- a/python/src/pywayang/platforms/python/channels.py +++ b/python/src/pywy/platforms/python/channels.py @@ -1,26 +1,5 @@ from typing import ( Iterable, Callable ) - -class Channel: - - def __init__(self): - pass - - def getchannel(self) -> 'Channel': - return self - - def gettype(self): - return type(self) - -class ChannelDescriptor: - - def __init__(self, channelType: type, isReusable: bool, isSuitableForBreakpoint: bool): - self.channelType = channelType - self.isReusable = isReusable - self.isSuitableForBreakpoint = isSuitableForBreakpoint - - def create_instance(self) -> Channel: - return self.channelType() - +from pywy.platforms.basic.channel import ( Channel, ChannelDescriptor ) class PyIteratorChannel(Channel): diff --git a/python/src/pywayang/platforms/python/compiler/__init__.py b/python/src/pywy/platforms/python/compiler/__init__.py similarity index 100% rename from python/src/pywayang/platforms/python/compiler/__init__.py rename to python/src/pywy/platforms/python/compiler/__init__.py diff --git a/python/src/pywayang/platforms/python/execution/__init__.py b/python/src/pywy/platforms/python/execution/__init__.py similarity index 100% rename from python/src/pywayang/platforms/python/execution/__init__.py rename to python/src/pywy/platforms/python/execution/__init__.py diff --git a/python/src/pywy/platforms/python/mappings.py b/python/src/pywy/platforms/python/mappings.py new file mode 100644 index 00000000..1c2783aa --- /dev/null +++ b/python/src/pywy/platforms/python/mappings.py @@ -0,0 +1,10 @@ +from pywy.platforms.basic.mapping import Mapping +from pywy.platforms.python.operators import * + + +PywyOperatorMappings = Mapping() + +PywyOperatorMappings.add_mapping(PyFilterOperator()) +PywyOperatorMappings.add_mapping(PyTextFileSourceOperator()) +PywyOperatorMappings.add_mapping(PyTextFileSinkOperator()) + diff --git a/python/src/pywayang/platforms/python/operators/PyFilterOperator.py b/python/src/pywy/platforms/python/operators/PyFilterOperator.py similarity index 78% rename from python/src/pywayang/platforms/python/operators/PyFilterOperator.py rename to python/src/pywy/platforms/python/operators/PyFilterOperator.py index f1d7dcb4..24b97d70 100644 --- a/python/src/pywayang/platforms/python/operators/PyFilterOperator.py +++ b/python/src/pywy/platforms/python/operators/PyFilterOperator.py @@ -1,8 +1,8 @@ -from pywayang.operator.unary import FilterOperator -from pywayang.platforms.python.operators.PythonExecutionOperator import PythonExecutionOperator -from pywayang.platforms.python.channels import (Channel, ChannelDescriptor, PyIteratorChannel, - PyIteratorChannelDescriptor, PyCallableChannelDescriptor, - PyCallableChannel) +from pywy.wayangplan.unary import FilterOperator +from pywy.platforms.python.operators.PythonExecutionOperator import PythonExecutionOperator +from pywy.platforms.python.channels import (Channel, ChannelDescriptor, PyIteratorChannel, + PyIteratorChannelDescriptor, PyCallableChannelDescriptor, + PyCallableChannel) from typing import Set class PyFilterOperator(FilterOperator, PythonExecutionOperator): diff --git a/python/src/pywayang/platforms/python/operators/PyTextFileSinkOperator.py b/python/src/pywy/platforms/python/operators/PyTextFileSinkOperator.py similarity index 87% rename from python/src/pywayang/platforms/python/operators/PyTextFileSinkOperator.py rename to python/src/pywy/platforms/python/operators/PyTextFileSinkOperator.py index 6589a634..8b244119 100644 --- a/python/src/pywayang/platforms/python/operators/PyTextFileSinkOperator.py +++ b/python/src/pywy/platforms/python/operators/PyTextFileSinkOperator.py @@ -1,6 +1,6 @@ -from pywayang.operator.sink import TextFileSink -from pywayang.platforms.python.operators.PythonExecutionOperator import PythonExecutionOperator -from pywayang.platforms.python.channels import ( +from pywy.wayangplan.sink import TextFileSink +from pywy.platforms.python.operators.PythonExecutionOperator import PythonExecutionOperator +from pywy.platforms.python.channels import ( Channel, ChannelDescriptor, PyIteratorChannel, diff --git a/python/src/pywayang/platforms/python/operators/PyTextFileSourceOperator.py b/python/src/pywy/platforms/python/operators/PyTextFileSourceOperator.py similarity index 87% rename from python/src/pywayang/platforms/python/operators/PyTextFileSourceOperator.py rename to python/src/pywy/platforms/python/operators/PyTextFileSourceOperator.py index ccfbec48..d228b0ef 100644 --- a/python/src/pywayang/platforms/python/operators/PyTextFileSourceOperator.py +++ b/python/src/pywy/platforms/python/operators/PyTextFileSourceOperator.py @@ -1,6 +1,6 @@ -from pywayang.operator.source import TextFileSource -from pywayang.platforms.python.operators.PythonExecutionOperator import PythonExecutionOperator -from pywayang.platforms.python.channels import ( +from pywy.wayangplan.source import TextFileSource +from pywy.platforms.python.operators.PythonExecutionOperator import PythonExecutionOperator +from pywy.platforms.python.channels import ( Channel, ChannelDescriptor, PyIteratorChannel, diff --git a/python/src/pywy/platforms/python/operators/PythonExecutionOperator.py b/python/src/pywy/platforms/python/operators/PythonExecutionOperator.py new file mode 100644 index 00000000..5dd5f7f7 --- /dev/null +++ b/python/src/pywy/platforms/python/operators/PythonExecutionOperator.py @@ -0,0 +1,7 @@ +from pywy.wayangplan.base import WyOperator +from pywy.platforms.python.channels import Channel + +class PythonExecutionOperator(WyOperator): + + def execute(self, inputs: Channel, output: Channel): + pass \ No newline at end of file diff --git a/python/src/pywy/platforms/python/operators/__init__.py b/python/src/pywy/platforms/python/operators/__init__.py new file mode 100644 index 00000000..5ddf61ae --- /dev/null +++ b/python/src/pywy/platforms/python/operators/__init__.py @@ -0,0 +1,11 @@ +from pywy.platforms.python.operators.PythonExecutionOperator import PythonExecutionOperator +from pywy.platforms.python.operators.PyFilterOperator import PyFilterOperator +from pywy.platforms.python.operators.PyTextFileSourceOperator import PyTextFileSourceOperator +from pywy.platforms.python.operators.PyTextFileSinkOperator import PyTextFileSinkOperator + +__ALL__ = [ + PythonExecutionOperator, + PyFilterOperator, + PyTextFileSourceOperator, + PyTextFileSinkOperator +] \ No newline at end of file diff --git a/python/src/pywy/platforms/python/platform/__init__.py b/python/src/pywy/platforms/python/platform/__init__.py new file mode 100644 index 00000000..f7ba3513 --- /dev/null +++ b/python/src/pywy/platforms/python/platform/__init__.py @@ -0,0 +1 @@ +from pywy.platforms.python.platform.platform import PythonPlatform \ No newline at end of file diff --git a/python/src/pywy/platforms/python/platform/platform.py b/python/src/pywy/platforms/python/platform/platform.py new file mode 100644 index 00000000..c4faddb0 --- /dev/null +++ b/python/src/pywy/platforms/python/platform/platform.py @@ -0,0 +1,6 @@ +from pywy.platforms.basic.platform import Platform + +class PythonPlatform(Platform): + + def __init__(self): + super(PythonPlatform, self).__init__("Python") \ No newline at end of file diff --git a/python/src/pywy/platforms/python/plugin/__init__.py b/python/src/pywy/platforms/python/plugin/__init__.py new file mode 100644 index 00000000..8c6b17ee --- /dev/null +++ b/python/src/pywy/platforms/python/plugin/__init__.py @@ -0,0 +1 @@ +from pywy.platforms.python.plugin.plugin import PythonPlugin \ No newline at end of file diff --git a/python/src/pywy/platforms/python/plugin/plugin.py b/python/src/pywy/platforms/python/plugin/plugin.py new file mode 100644 index 00000000..0f870029 --- /dev/null +++ b/python/src/pywy/platforms/python/plugin/plugin.py @@ -0,0 +1,10 @@ +from pywy.platforms.python.platform import PythonPlatform +from pywy.platforms.basic.plugin import Plugin +from pywy.platforms.python.mappings import PywyOperatorMappings + + + +class PythonPlugin(Plugin): + + def __init__(self): + super(PythonPlugin, self).__init__(PythonPlatform(), PywyOperatorMappings) \ No newline at end of file diff --git a/python/src/pywy/plugins.py b/python/src/pywy/plugins.py new file mode 100644 index 00000000..1e55e310 --- /dev/null +++ b/python/src/pywy/plugins.py @@ -0,0 +1,10 @@ +from pywy.platforms.basic.platform import Platform +from pywy.platforms.basic.plugin import Plugin +from pywy.platforms.python.plugin import PythonPlugin + +# define the basic plugins that can be used +java = Plugin(Platform('java')) +spark = Plugin(Platform('spark')) +flink = Plugin(Platform('flink')) +# plugin for the python platform +python = PythonPlugin() diff --git a/python/src/pywayang/test.py b/python/src/pywy/test.py similarity index 70% rename from python/src/pywayang/test.py rename to python/src/pywy/test.py index 884acfc5..ec4fbad0 100644 --- a/python/src/pywayang/test.py +++ b/python/src/pywy/test.py @@ -1,10 +1,8 @@ -from typing import Iterable - -from pywayang.platform import Platform -from pywayang.context import WayangContext -from pywayang.platforms.python.channels import Channel -from pywayang.plugin import java, spark -from pywayang.operator.unary import * +from pywy.platforms.basic.platform import Platform +from pywy.context import WayangContext +from pywy.platforms.python.channels import Channel +from pywy.platforms.basic.plugin import java, spark +from pywy.wayangplan.unary import * p = Platform("nana") print("LALA "+str(p)) @@ -17,7 +15,7 @@ print(type(p2)) print(str(WayangContext().register(java, spark))) -from pywayang.types import Predicate, getTypePredicate +from pywy.types import Predicate, getTypePredicate predicate : Predicate = lambda x: x % 2 == 0 getTypePredicate(predicate) @@ -34,7 +32,9 @@ def fmfunc(i:int) -> str: fileop = WayangContext()\ .register(java)\ - .textFile("here")\ + .textFile("/Users/bertty/databloom/blossom/python/resources/test.input")\ + .filter(pre)\ + .storeTextFile("/Users/bertty/databloom/blossom/python/resources/test.output") filterop: FilterOperator = fileop.filter(pre).getOperator() #fop_pre = filterop.getWrapper() @@ -66,10 +66,10 @@ def concatenate(function_a, function_b): #for i in res_pro: # print(i) -from pywayang.platforms.python.mappings import OperatorMappings -from pywayang.platforms.python.operators import * +from pywy.platforms.python.mappings import PywyOperatorMappings +from pywy.platforms.python.operators import * -print(OperatorMappings) +print(PywyOperatorMappings) pyF = PyFilterOperator() print(pyF) @@ -92,6 +92,6 @@ print(ou1) for i in ou1: print(i) -pyFM = OperatorMappings.get_instanceof(filterop) +pyFM = PywyOperatorMappings.get_instanceof(filterop) print(pyFM) print(type(pyFM)) \ No newline at end of file diff --git a/python/src/pywayang/protobuf/__init__.py b/python/src/pywy/test/__init__.py similarity index 100% rename from python/src/pywayang/protobuf/__init__.py rename to python/src/pywy/test/__init__.py diff --git a/python/src/pywayang/test/demo_testing.py b/python/src/pywy/test/demo_testing.py similarity index 100% rename from python/src/pywayang/test/demo_testing.py rename to python/src/pywy/test/demo_testing.py diff --git a/python/src/pywayang/test/full_java_test.py b/python/src/pywy/test/full_java_test.py similarity index 95% rename from python/src/pywayang/test/full_java_test.py rename to python/src/pywy/test/full_java_test.py index 4b2f034b..e1e512b1 100644 --- a/python/src/pywayang/test/full_java_test.py +++ b/python/src/pywy/test/full_java_test.py @@ -16,8 +16,8 @@ # import unittest -from pywayang.orchestrator.plan import Descriptor -from pywayang.orchestrator.dataquanta import DataQuantaBuilder +from pywy.orchestrator.plan import Descriptor +from pywy.orchestrator.dataquanta import DataQuantaBuilder input_1 = "./../../../resources/text.input" input_2 = "./../../../resources/text.input" diff --git a/python/src/pywayang/test/full_spark_test.py b/python/src/pywy/test/full_spark_test.py similarity index 95% rename from python/src/pywayang/test/full_spark_test.py rename to python/src/pywy/test/full_spark_test.py index c3d72f15..e32a1c2d 100644 --- a/python/src/pywayang/test/full_spark_test.py +++ b/python/src/pywy/test/full_spark_test.py @@ -15,8 +15,8 @@ # limitations under the License. # import unittest -from pywayang.orchestrator.plan import Descriptor -from pywayang.orchestrator.dataquanta import DataQuantaBuilder +from pywy.orchestrator.plan import Descriptor +from pywy.orchestrator.dataquanta import DataQuantaBuilder def test_most_basic(self): diff --git a/python/src/pywayang/platforms/python/platform/__init__.py b/python/src/pywy/translate/__init__.py similarity index 100% rename from python/src/pywayang/platforms/python/platform/__init__.py rename to python/src/pywy/translate/__init__.py diff --git a/python/src/pywayang/test/__init__.py b/python/src/pywy/translate/protobuf/__init__.py similarity index 100% rename from python/src/pywayang/test/__init__.py rename to python/src/pywy/translate/protobuf/__init__.py diff --git a/python/src/pywayang/protobuf/planwriter.py b/python/src/pywy/translate/protobuf/planwriter.py similarity index 99% rename from python/src/pywayang/protobuf/planwriter.py rename to python/src/pywy/translate/protobuf/planwriter.py index 14fc259b..7f133128 100644 --- a/python/src/pywayang/protobuf/planwriter.py +++ b/python/src/pywy/translate/protobuf/planwriter.py @@ -15,11 +15,10 @@ # limitations under the License. # -import pywayang.protobuf.pywayangplan_pb2 as pwb +import pywy.translate.protobuf.pywayangplan_pb2 as pwb import os import cloudpickle import logging -import pathlib import requests import base64 diff --git a/python/src/pywy/translate/translator.py b/python/src/pywy/translate/translator.py new file mode 100644 index 00000000..303d497b --- /dev/null +++ b/python/src/pywy/translate/translator.py @@ -0,0 +1,12 @@ +from pywy.platforms.basic.plugin import Plugin +from pywy.wayangplan.wayang import WayangPlan +from pywy.platforms.basic.mapping import Mapping + +class Translator: + + def __init__(self, plugin: Plugin, plan: WayangPlan): + self.plugin = plugin + self.plan = plan + + def translate(self): + mappings:Mapping = self.plugin.get_mappings() diff --git a/python/src/pywayang/types.py b/python/src/pywy/types.py similarity index 100% rename from python/src/pywayang/types.py rename to python/src/pywy/types.py diff --git a/python/src/pywy/wayangplan/__init__.py b/python/src/pywy/wayangplan/__init__.py new file mode 100644 index 00000000..dae59871 --- /dev/null +++ b/python/src/pywy/wayangplan/__init__.py @@ -0,0 +1,15 @@ +from pywy.wayangplan.wayang import WayangPlan +from pywy.wayangplan.base import WyOperator +from pywy.wayangplan.sink import TextFileSink +from pywy.wayangplan.source import TextFileSource +from pywy.wayangplan.unary import FilterOperator, MapOperator, FlatmapOperator + +__ALL__= [ + WayangPlan, + WyOperator, + TextFileSink, + TextFileSource, + FilterOperator, + MapOperator, + FlatmapOperator +] \ No newline at end of file diff --git a/python/src/pywayang/operator/base.py b/python/src/pywy/wayangplan/base.py similarity index 96% rename from python/src/pywayang/operator/base.py rename to python/src/pywy/wayangplan/base.py index b7834d17..90e0b622 100644 --- a/python/src/pywayang/operator/base.py +++ b/python/src/pywy/wayangplan/base.py @@ -1,5 +1,5 @@ from typing import (TypeVar, Optional, List, Set) -from pywayang.platforms.python.channels import ChannelDescriptor +from pywy.platforms.basic.channel import ChannelDescriptor class WyOperator: diff --git a/python/src/pywayang/operator/sink.py b/python/src/pywy/wayangplan/sink.py similarity index 55% rename from python/src/pywayang/operator/sink.py rename to python/src/pywy/wayangplan/sink.py index 52cbeb0e..89443248 100644 --- a/python/src/pywayang/operator/sink.py +++ b/python/src/pywy/wayangplan/sink.py @@ -1,6 +1,17 @@ -from pywayang.operator.base import WyOperator +from pywy.wayangplan.base import WyOperator -class SinkUnaryOperator(WyOperator): +class SinkOperator(WyOperator): + + def __init__(self, name:str): + super().__init__(name, None, str, 0, 1) + + def __str__(self): + return super().__str__() + + def __repr__(self): + return super().__repr__() + +class SinkUnaryOperator(SinkOperator): def __init__(self, name:str): super().__init__(name, None, str, 0, 1) diff --git a/python/src/pywayang/operator/source.py b/python/src/pywy/wayangplan/source.py similarity index 86% rename from python/src/pywayang/operator/source.py rename to python/src/pywy/wayangplan/source.py index 97aff75a..c9ae2ba1 100644 --- a/python/src/pywayang/operator/source.py +++ b/python/src/pywy/wayangplan/source.py @@ -1,4 +1,4 @@ -from pywayang.operator.base import WyOperator +from pywy.wayangplan.base import WyOperator class SourceUnaryOperator(WyOperator): diff --git a/python/src/pywayang/operator/unary.py b/python/src/pywy/wayangplan/unary.py similarity index 95% rename from python/src/pywayang/operator/unary.py rename to python/src/pywy/wayangplan/unary.py index 559effa5..a43b4c25 100644 --- a/python/src/pywayang/operator/unary.py +++ b/python/src/pywy/wayangplan/unary.py @@ -1,5 +1,5 @@ -from pywayang.operator.base import WyOperator -from pywayang.types import ( +from pywy.wayangplan.base import WyOperator +from pywy.types import ( GenericTco, GenericUco, Predicate, diff --git a/python/src/pywy/wayangplan/wayang.py b/python/src/pywy/wayangplan/wayang.py new file mode 100644 index 00000000..a08b572a --- /dev/null +++ b/python/src/pywy/wayangplan/wayang.py @@ -0,0 +1,11 @@ +from typing import Iterable + +from pywy.wayangplan.sink import SinkOperator +from pywy.platforms.basic.platform import Platform + + +class WayangPlan: + + def __init__(self, platforms: Iterable[Platform], sinks: Iterable[SinkOperator]): + self.platforms = platforms + self.sinks = sinks
