This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-python.git
The following commit(s) were added to refs/heads/main by this push:
new a7b752a Refactoring module structure and building (#15)
a7b752a is described below
commit a7b752af57cc844a957bb6d6aaafba4eeaae0d38
Author: yuzelin <[email protected]>
AuthorDate: Wed Oct 9 10:45:54 2024 +0800
Refactoring module structure and building (#15)
---
.github/workflows/paimon-python-checks.yml | 10 ++++--
.gitignore | 2 +-
README.md | 30 ++++++++--------
paimon_python_api/__init__.py | 25 ++++++++++++++
paimon_python_api/catalog.py | 2 +-
paimon_python_api/read_builder.py | 3 +-
paimon_python_api/table.py | 3 +-
paimon_python_api/table_commit.py | 2 +-
paimon_python_api/table_read.py | 7 ++--
paimon_python_api/table_scan.py | 2 +-
paimon_python_api/table_write.py | 7 ++--
paimon_python_api/write_builder.py | 3 +-
.../util => paimon_python_java}/__init__.py | 19 ++++++++++
.../gateway_server.py | 30 ++++++++++++----
.../java_gateway.py | 8 ++---
.../paimon-python-java-bridge/copyright.txt | 0
.../paimon-python-java-bridge/pom.xml | 22 ++++++++++--
.../java/org/apache/paimon/python/BytesWriter.java | 0
.../java/org/apache/paimon/python/FileLock.java | 0
.../org/apache/paimon/python/InvocationUtil.java | 0
.../java/org/apache/paimon/python/NetUtils.java | 0
.../apache/paimon/python/ParallelBytesReader.java | 0
.../org/apache/paimon/python/PythonEnvUtils.java | 0
.../apache/paimon/python/PythonGatewayServer.java | 0
.../apache/paimon/python/RecordBytesIterator.java | 0
.../tools/maven/checkstyle.xml | 0
.../tools/maven/suppressions.xml | 0
.../api_impl.py => paimon_python_java/pypaimon.py | 27 +++++++--------
.../tests}/__init__.py | 0
.../tests/test_write_and_read.py | 13 +++----
.../tests/utils.py | 6 ++--
.../tests => paimon_python_java/util}/__init__.py | 0
.../util/constants.py | 1 -
.../util/exceptions.py | 0
.../util/java_utils.py | 2 +-
setup.py | 20 +++++++----
.../util => setup_utils}/__init__.py | 2 ++
.../java_setuputils.py | 40 ++++++++++------------
tox.ini | 1 +
39 files changed, 186 insertions(+), 101 deletions(-)
diff --git a/.github/workflows/paimon-python-checks.yml
b/.github/workflows/paimon-python-checks.yml
index 1e85b1b..e94820b 100644
--- a/.github/workflows/paimon-python-checks.yml
+++ b/.github/workflows/paimon-python-checks.yml
@@ -22,10 +22,11 @@ on:
push:
pull_request:
paths-ignore:
- - 'dev/**'
- - 'java_based_implementation/paimon-python-java-bridge/**'
- '**/*.md'
+env:
+ JDK_VERSION: 8
+
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{
github.event.number || github.run_id }}
cancel-in-progress: true
@@ -37,6 +38,11 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v2
+ - name: Set up JDK ${{ env.JDK_VERSION }}
+ uses: actions/setup-java@v2
+ with:
+ java-version: ${{ env.JDK_VERSION }}
+ distribution: 'adopt'
- name: Run lint-python.sh
run: |
chmod +x dev/lint-python.sh
diff --git a/.gitignore b/.gitignore
index c84443a..08fac18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,4 +15,4 @@ dev/.stage.txt
dev/download
dev/log
dist/
-paimon_python.egg-info/
+*.egg-info/
diff --git a/README.md b/README.md
index f605935..cd2d360 100644
--- a/README.md
+++ b/README.md
@@ -22,30 +22,28 @@ This repo is for Apache Paimon Python SDK.
You can verify the setup by right-clicking on any file or folder in the
flink-python project
and running "External Tools" → "flake8".
-# Usage
-
-## Java-Based Implementation
+## Check
-We can use `py4j` to leverage Java code to read Paimon data. This section
describes how to use this implementation.
+We provide script to check codes.
-### Set Environment Variables
+```shell
+./dev/lint-python.sh # execute all checks
+./dev/lint-python.sh -h # run this to see more usages
+```
-`py4j` need to access a JVM, so we should set JVM arguments (optional) and
Java classpath. A convenient way is using
-`os` packages to set environment variables which only affect current process.
+## Build
-```python
-import os
+We provide script to build wheel.
-os.environ['PYPAIMON_JAVA_CLASSPATH'] = '/path/to/dependent_jars/*'
-os.environ['_PYPAIMON_JVM_ARGS'] = 'jvm_arg1 jvm_arg2 ...'
+```shell
+./dev/build-wheels.sh
```
-NOTE: the package has set paimon core and hadoop dependencies. If you just
test in local or run code in hadoop, you doesn't
-need to set classpath. If you need other dependencies such as OSS/S3
filesystem jars, or special catalog which isn't implemented
-in paimon core, please download jars and set classpath.
+The target wheel is under `dist/`
+
+# Usage
-# API Reference
-TODO
+See Apache Paimon Python API
[Doc](https://paimon.apache.org/docs/master/program-api/python-api/).
diff --git a/paimon_python_api/__init__.py b/paimon_python_api/__init__.py
index e154fad..b3842d0 100644
--- a/paimon_python_api/__init__.py
+++ b/paimon_python_api/__init__.py
@@ -15,3 +15,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################
+
+from .split import Split
+from .table_read import TableRead
+from .table_scan import TableScan, Plan
+from .read_builder import ReadBuilder
+from .commit_message import CommitMessage
+from .table_commit import BatchTableCommit
+from .table_write import BatchTableWrite
+from .write_builder import BatchWriteBuilder
+from .table import Table
+from .catalog import Catalog
+
+__all__ = [
+ 'Split',
+ 'TableRead',
+ 'TableScan',
+ 'Plan',
+ 'ReadBuilder',
+ 'CommitMessage',
+ 'BatchTableCommit',
+ 'BatchTableWrite',
+ 'BatchWriteBuilder',
+ 'Table',
+ 'Catalog'
+]
diff --git a/paimon_python_api/catalog.py b/paimon_python_api/catalog.py
index 7fbacc5..412e9f0 100644
--- a/paimon_python_api/catalog.py
+++ b/paimon_python_api/catalog.py
@@ -17,7 +17,7 @@
#################################################################################
from abc import ABC, abstractmethod
-from paimon_python_api.table import Table
+from paimon_python_api import Table
class Catalog(ABC):
diff --git a/paimon_python_api/read_builder.py
b/paimon_python_api/read_builder.py
index bc4266d..94ec073 100644
--- a/paimon_python_api/read_builder.py
+++ b/paimon_python_api/read_builder.py
@@ -17,8 +17,7 @@
#################################################################################
from abc import ABC, abstractmethod
-from paimon_python_api.table_read import TableRead
-from paimon_python_api.table_scan import TableScan
+from paimon_python_api import TableRead, TableScan
from typing import List
diff --git a/paimon_python_api/table.py b/paimon_python_api/table.py
index e1bacd7..35b81ac 100644
--- a/paimon_python_api/table.py
+++ b/paimon_python_api/table.py
@@ -17,8 +17,7 @@
#################################################################################
from abc import ABC, abstractmethod
-from paimon_python_api.read_builder import ReadBuilder
-from paimon_python_api.write_builder import BatchWriteBuilder
+from paimon_python_api import ReadBuilder, BatchWriteBuilder
class Table(ABC):
diff --git a/paimon_python_api/table_commit.py
b/paimon_python_api/table_commit.py
index 517b66e..d76ed6a 100644
--- a/paimon_python_api/table_commit.py
+++ b/paimon_python_api/table_commit.py
@@ -17,7 +17,7 @@
#################################################################################
from abc import ABC, abstractmethod
-from paimon_python_api.commit_message import CommitMessage
+from paimon_python_api import CommitMessage
from typing import List
diff --git a/paimon_python_api/table_read.py b/paimon_python_api/table_read.py
index 47b8aae..4fa12c1 100644
--- a/paimon_python_api/table_read.py
+++ b/paimon_python_api/table_read.py
@@ -16,9 +16,10 @@
# limitations under the License.
#################################################################################
+import pyarrow as pa
+
from abc import ABC, abstractmethod
-from pyarrow import RecordBatchReader
-from paimon_python_api.split import Split
+from paimon_python_api import Split
from typing import List
@@ -26,5 +27,5 @@ class TableRead(ABC):
"""To read data from data splits."""
@abstractmethod
- def create_reader(self, splits: List[Split]) -> RecordBatchReader:
+ def create_reader(self, splits: List[Split]) -> pa.RecordBatchReader:
"""Return a reader containing batches of pyarrow format."""
diff --git a/paimon_python_api/table_scan.py b/paimon_python_api/table_scan.py
index 8e29c12..d998462 100644
--- a/paimon_python_api/table_scan.py
+++ b/paimon_python_api/table_scan.py
@@ -18,7 +18,7 @@
from abc import ABC, abstractmethod
from typing import List
-from paimon_python_api.split import Split
+from paimon_python_api import Split
class TableScan(ABC):
diff --git a/paimon_python_api/table_write.py b/paimon_python_api/table_write.py
index d0052a1..167ceeb 100644
--- a/paimon_python_api/table_write.py
+++ b/paimon_python_api/table_write.py
@@ -16,9 +16,10 @@
# limitations under the License.
#################################################################################
+import pyarrow as pa
+
from abc import ABC, abstractmethod
-from paimon_python_api.commit_message import CommitMessage
-from pyarrow import RecordBatch
+from paimon_python_api import CommitMessage
from typing import List
@@ -26,7 +27,7 @@ class BatchTableWrite(ABC):
"""A table write for batch processing. Recommended for one-time
committing."""
@abstractmethod
- def write(self, record_batch: RecordBatch):
+ def write(self, record_batch: pa.RecordBatch):
""" Write a batch to the writer. */"""
@abstractmethod
diff --git a/paimon_python_api/write_builder.py
b/paimon_python_api/write_builder.py
index b36fdbe..caa23b9 100644
--- a/paimon_python_api/write_builder.py
+++ b/paimon_python_api/write_builder.py
@@ -17,8 +17,7 @@
#################################################################################
from abc import ABC, abstractmethod
-from paimon_python_api.table_commit import BatchTableCommit
-from paimon_python_api.table_write import BatchTableWrite
+from paimon_python_api import BatchTableCommit, BatchTableWrite
class BatchWriteBuilder(ABC):
diff --git a/java_based_implementation/util/__init__.py
b/paimon_python_java/__init__.py
similarity index 68%
copy from java_based_implementation/util/__init__.py
copy to paimon_python_java/__init__.py
index 65b48d4..6e97d9e 100644
--- a/java_based_implementation/util/__init__.py
+++ b/paimon_python_java/__init__.py
@@ -15,3 +15,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
+
+from .util import constants
+from .pypaimon import (Catalog, Table, ReadBuilder, TableScan, Plan, Split,
TableRead,
+ BatchWriteBuilder, BatchTableWrite, CommitMessage,
BatchTableCommit)
+
+__all__ = [
+ 'constants',
+ 'Catalog',
+ 'Table',
+ 'ReadBuilder',
+ 'TableScan',
+ 'Plan',
+ 'Split',
+ 'TableRead',
+ 'BatchWriteBuilder',
+ 'BatchTableWrite',
+ 'CommitMessage',
+ 'BatchTableCommit'
+]
diff --git a/java_based_implementation/gateway_server.py
b/paimon_python_java/gateway_server.py
similarity index 74%
rename from java_based_implementation/gateway_server.py
rename to paimon_python_java/gateway_server.py
index 60940a5..31933e2 100644
--- a/java_based_implementation/gateway_server.py
+++ b/paimon_python_java/gateway_server.py
@@ -16,14 +16,13 @@
# limitations under the License.
################################################################################
+import importlib
import os
import platform
import signal
from subprocess import Popen, PIPE
-from java_based_implementation.util.constants import (PYPAIMON_JVM_ARGS,
PYPAIMON_MAIN_ARGS,
- PYPAIMON_MAIN_CLASS)
-from java_based_implementation.util.setup_utils import get_classpath
+from paimon_python_java import constants
def on_windows():
@@ -47,9 +46,9 @@ def launch_gateway_server_process(env):
java_executable = find_java_executable()
# TODO construct Java module log settings
log_settings = []
- jvm_args = env.get(PYPAIMON_JVM_ARGS, '').split()
- classpath = get_classpath(env)
- main_args = env.get(PYPAIMON_MAIN_ARGS, '').split()
+ jvm_args = env.get(constants.PYPAIMON_JVM_ARGS, '').split()
+ classpath = _get_classpath(env)
+ main_args = env.get(constants.PYPAIMON_MAIN_ARGS, '').split()
command = [
java_executable,
*jvm_args,
@@ -60,7 +59,7 @@ def launch_gateway_server_process(env):
"-cp",
classpath,
"-c",
- PYPAIMON_MAIN_CLASS,
+ constants.PYPAIMON_MAIN_CLASS,
*main_args
]
@@ -72,3 +71,20 @@ def launch_gateway_server_process(env):
preexec_fn = preexec_func
return Popen(list(filter(lambda c: len(c) != 0, command)),
stdin=PIPE, stderr=PIPE, preexec_fn=preexec_fn, env=env)
+
+
+_JAVA_IMPL_MODULE = 'paimon_python_java'
+_JAVA_DEPS = 'java_dependencies'
+_JAVA_BRIDGE = 'paimon-python-java-bridge'
+
+
+def _get_classpath(env):
+ user_defined = env.get(constants.PYPAIMON_JAVA_CLASSPATH)
+
+ module = importlib.import_module(_JAVA_IMPL_MODULE)
+ builtin_java_bridge = os.path.join(*module.__path__, _JAVA_DEPS,
_JAVA_BRIDGE + '.jar')
+
+ if user_defined is None:
+ return builtin_java_bridge
+ else:
+ return os.pathsep.join([builtin_java_bridge, user_defined])
diff --git a/java_based_implementation/java_gateway.py
b/paimon_python_java/java_gateway.py
similarity index 93%
rename from java_based_implementation/java_gateway.py
rename to paimon_python_java/java_gateway.py
index 64f40b4..372ae4a 100644
--- a/java_based_implementation/java_gateway.py
+++ b/paimon_python_java/java_gateway.py
@@ -26,9 +26,9 @@ from py4j.java_gateway import (java_import, logger,
JavaGateway, GatewayParamete
CallbackServerParameters)
from threading import RLock
-from java_based_implementation.gateway_server import
launch_gateway_server_process
-from java_based_implementation.util.constants import PYPAIMON_CONN_INFO_PATH
-from java_based_implementation.util.exceptions import install_py4j_hooks
+from paimon_python_java.gateway_server import launch_gateway_server_process
+from paimon_python_java import constants
+from paimon_python_java.util.exceptions import install_py4j_hooks
_gateway = None
_lock = RLock()
@@ -73,7 +73,7 @@ def launch_gateway():
os.unlink(conn_info_file)
env = dict(os.environ)
- env[PYPAIMON_CONN_INFO_PATH] = conn_info_file
+ env[constants.PYPAIMON_CONN_INFO_PATH] = conn_info_file
p = launch_gateway_server_process(env)
diff --git a/java_based_implementation/paimon-python-java-bridge/copyright.txt
b/paimon_python_java/paimon-python-java-bridge/copyright.txt
similarity index 100%
rename from java_based_implementation/paimon-python-java-bridge/copyright.txt
rename to paimon_python_java/paimon-python-java-bridge/copyright.txt
diff --git a/java_based_implementation/paimon-python-java-bridge/pom.xml
b/paimon_python_java/paimon-python-java-bridge/pom.xml
similarity index 87%
rename from java_based_implementation/paimon-python-java-bridge/pom.xml
rename to paimon_python_java/paimon-python-java-bridge/pom.xml
index 5eb26e1..7fe5858 100644
--- a/java_based_implementation/paimon-python-java-bridge/pom.xml
+++ b/paimon_python_java/paimon-python-java-bridge/pom.xml
@@ -23,13 +23,13 @@
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-python-java-bridge</artifactId>
- <version>0.9-SNAPSHOT</version>
+ <version>0.1-SNAPSHOT</version>
<name>Paimon : Python-Java Bridge</name>
<packaging>jar</packaging>
<properties>
- <paimon.version>0.9-SNAPSHOT</paimon.version>
+ <paimon.version>0.9.0</paimon.version>
<flink.shaded.hadoop.version>2.8.3-10.0</flink.shaded.hadoop.version>
<py4j.version>0.10.9.7</py4j.version>
<slf4j.version>1.7.32</slf4j.version>
@@ -37,6 +37,7 @@
<spotless.version>2.13.0</spotless.version>
<spotless.delimiter>package</spotless.delimiter>
<arrow.version>14.0.0</arrow.version>
+ <target.java.version>1.8</target.java.version>
</properties>
<dependencies>
@@ -137,6 +138,23 @@
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.0</version>
+ <configuration>
+ <source>${target.java.version}</source>
+ <target>${target.java.version}</target>
+ <!-- The semantics of this option are reversed, see
MCOMPILER-209. -->
+
<useIncrementalCompilation>false</useIncrementalCompilation>
+ <compilerArgs>
+ <!-- Prevents recompilation due to missing
package-info.class, see MCOMPILER-205 -->
+ <arg>-Xpkginfo:always</arg>
+ <arg>-Xlint:deprecation</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
diff --git
a/java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java
b/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java
rename to
paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/BytesWriter.java
diff --git
a/java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java
b/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java
rename to
paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/FileLock.java
diff --git
a/java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java
b/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java
rename to
paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/InvocationUtil.java
diff --git
a/java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java
b/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java
rename to
paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/NetUtils.java
diff --git
a/java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java
b/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java
rename to
paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/ParallelBytesReader.java
diff --git
a/java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java
b/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java
rename to
paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonEnvUtils.java
diff --git
a/java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java
b/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java
rename to
paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/PythonGatewayServer.java
diff --git
a/java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java
b/paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java
rename to
paimon_python_java/paimon-python-java-bridge/src/main/java/org/apache/paimon/python/RecordBytesIterator.java
diff --git
a/java_based_implementation/paimon-python-java-bridge/tools/maven/checkstyle.xml
b/paimon_python_java/paimon-python-java-bridge/tools/maven/checkstyle.xml
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/tools/maven/checkstyle.xml
rename to
paimon_python_java/paimon-python-java-bridge/tools/maven/checkstyle.xml
diff --git
a/java_based_implementation/paimon-python-java-bridge/tools/maven/suppressions.xml
b/paimon_python_java/paimon-python-java-bridge/tools/maven/suppressions.xml
similarity index 100%
rename from
java_based_implementation/paimon-python-java-bridge/tools/maven/suppressions.xml
rename to
paimon_python_java/paimon-python-java-bridge/tools/maven/suppressions.xml
diff --git a/java_based_implementation/api_impl.py
b/paimon_python_java/pypaimon.py
similarity index 89%
rename from java_based_implementation/api_impl.py
rename to paimon_python_java/pypaimon.py
index 05487f4..ee43a17 100644
--- a/java_based_implementation/api_impl.py
+++ b/paimon_python_java/pypaimon.py
@@ -16,13 +16,12 @@
# limitations under the License.
################################################################################
-from java_based_implementation.java_gateway import get_gateway
-from java_based_implementation.util import constants
-from java_based_implementation.util.java_utils import to_j_catalog_context,
check_batch_write
+import pyarrow as pa
+
+from paimon_python_java.java_gateway import get_gateway
+from paimon_python_java.util import java_utils, constants
from paimon_python_api import (catalog, table, read_builder, table_scan,
split, table_read,
write_builder, table_write, commit_message,
table_commit)
-from pyarrow import (RecordBatch, BufferOutputStream, RecordBatchStreamWriter,
- RecordBatchStreamReader, BufferReader, RecordBatchReader)
from typing import List, Iterator
@@ -34,7 +33,7 @@ class Catalog(catalog.Catalog):
@staticmethod
def create(catalog_options: dict) -> 'Catalog':
- j_catalog_context = to_j_catalog_context(catalog_options)
+ j_catalog_context = java_utils.to_j_catalog_context(catalog_options)
gateway = get_gateway()
j_catalog = gateway.jvm.CatalogFactory.createCatalog(j_catalog_context)
return Catalog(j_catalog, catalog_options)
@@ -57,7 +56,7 @@ class Table(table.Table):
return ReadBuilder(j_read_builder, self._j_table.rowType(),
self._catalog_options)
def new_batch_write_builder(self) -> 'BatchWriteBuilder':
- check_batch_write(self._j_table)
+ java_utils.check_batch_write(self._j_table)
j_batch_write_builder =
get_gateway().jvm.InvocationUtil.getBatchWriteBuilder(self._j_table)
return BatchWriteBuilder(j_batch_write_builder,
self._j_table.rowType())
@@ -129,7 +128,7 @@ class TableRead(table_read.TableRead):
j_splits = list(map(lambda s: s.to_j_split(), splits))
self._j_bytes_reader.setSplits(j_splits)
batch_iterator = self._batch_generator()
- return RecordBatchReader.from_batches(self._arrow_schema,
batch_iterator)
+ return pa.RecordBatchReader.from_batches(self._arrow_schema,
batch_iterator)
def _init(self):
if self._j_bytes_reader is None:
@@ -147,17 +146,17 @@ class TableRead(table_read.TableRead):
if self._arrow_schema is None:
schema_bytes = self._j_bytes_reader.serializeSchema()
- schema_reader = RecordBatchStreamReader(BufferReader(schema_bytes))
+ schema_reader =
pa.RecordBatchStreamReader(pa.BufferReader(schema_bytes))
self._arrow_schema = schema_reader.schema
schema_reader.close()
- def _batch_generator(self) -> Iterator[RecordBatch]:
+ def _batch_generator(self) -> Iterator[pa.RecordBatch]:
while True:
next_bytes = self._j_bytes_reader.next()
if next_bytes is None:
break
else:
- stream_reader =
RecordBatchStreamReader(BufferReader(next_bytes))
+ stream_reader =
pa.RecordBatchStreamReader(pa.BufferReader(next_bytes))
yield from stream_reader
@@ -187,9 +186,9 @@ class BatchTableWrite(table_write.BatchTableWrite):
self._j_bytes_writer =
get_gateway().jvm.InvocationUtil.createBytesWriter(
j_batch_table_write, j_row_type)
- def write(self, record_batch: RecordBatch):
- stream = BufferOutputStream()
- with RecordBatchStreamWriter(stream, record_batch.schema) as writer:
+ def write(self, record_batch: pa.RecordBatch):
+ stream = pa.BufferOutputStream()
+ with pa.RecordBatchStreamWriter(stream, record_batch.schema) as writer:
writer.write(record_batch)
writer.close()
arrow_bytes = stream.getvalue().to_pybytes()
diff --git a/java_based_implementation/__init__.py
b/paimon_python_java/tests/__init__.py
similarity index 100%
rename from java_based_implementation/__init__.py
rename to paimon_python_java/tests/__init__.py
diff --git a/java_based_implementation/tests/test_write_and_read.py
b/paimon_python_java/tests/test_write_and_read.py
similarity index 96%
rename from java_based_implementation/tests/test_write_and_read.py
rename to paimon_python_java/tests/test_write_and_read.py
index 0ea834e..19c9ea8 100644
--- a/java_based_implementation/tests/test_write_and_read.py
+++ b/paimon_python_java/tests/test_write_and_read.py
@@ -20,11 +20,12 @@ import tempfile
import unittest
import pandas as pd
import pyarrow as pa
+import setup_utils.java_setuputils as setuputils
-from java_based_implementation.api_impl import Catalog, Table
-from java_based_implementation.java_gateway import get_gateway
-from java_based_implementation.tests.utils import create_simple_table
-from java_based_implementation.util import java_utils, setup_utils
+from paimon_python_java import Catalog, Table
+from paimon_python_java.java_gateway import get_gateway
+from paimon_python_java.tests.utils import create_simple_table
+from paimon_python_java.util import java_utils
from py4j.protocol import Py4JJavaError
@@ -32,12 +33,12 @@ class TableWriteReadTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
- setup_utils.setup_java_bridge()
+ setuputils.setup_java_bridge()
cls.warehouse = tempfile.mkdtemp()
@classmethod
def tearDownClass(cls):
- setup_utils.clean()
+ setuputils.clean()
def testReadEmptyAppendTable(self):
create_simple_table(self.warehouse, 'default', 'empty_append_table',
False)
diff --git a/java_based_implementation/tests/utils.py
b/paimon_python_java/tests/utils.py
similarity index 89%
rename from java_based_implementation/tests/utils.py
rename to paimon_python_java/tests/utils.py
index e709e0a..e0a79d5 100644
--- a/java_based_implementation/tests/utils.py
+++ b/paimon_python_java/tests/utils.py
@@ -16,8 +16,8 @@
# limitations under the License.
################################################################################
-from java_based_implementation.java_gateway import get_gateway
-from java_based_implementation.util.java_utils import to_j_catalog_context
+from paimon_python_java.java_gateway import get_gateway
+from paimon_python_java.util import java_utils
def create_simple_table(warehouse, database, table_name, has_pk, options=None):
@@ -29,7 +29,7 @@ def create_simple_table(warehouse, database, table_name,
has_pk, options=None):
gateway = get_gateway()
- j_catalog_context = to_j_catalog_context({'warehouse': warehouse})
+ j_catalog_context = java_utils.to_j_catalog_context({'warehouse':
warehouse})
j_catalog = gateway.jvm.CatalogFactory.createCatalog(j_catalog_context)
j_schema_builder = (
diff --git a/java_based_implementation/tests/__init__.py
b/paimon_python_java/util/__init__.py
similarity index 100%
rename from java_based_implementation/tests/__init__.py
rename to paimon_python_java/util/__init__.py
diff --git a/java_based_implementation/util/constants.py
b/paimon_python_java/util/constants.py
similarity index 97%
rename from java_based_implementation/util/constants.py
rename to paimon_python_java/util/constants.py
index 7e8923a..00f419c 100644
--- a/java_based_implementation/util/constants.py
+++ b/paimon_python_java/util/constants.py
@@ -22,7 +22,6 @@ PYPAIMON_JVM_ARGS = '_PYPAIMON_JVM_ARGS'
PYPAIMON_JAVA_CLASSPATH = '_PYPAIMON_JAVA_CLASSPATH'
PYPAIMON_MAIN_CLASS = 'org.apache.paimon.python.PythonGatewayServer'
PYPAIMON_MAIN_ARGS = '_PYPAIMON_MAIN_ARGS'
-PYPAIMON_TOX_TEST = '_PYPAIMON_TOX_TEST'
# ------------------------ for catalog options ------------------------
MAX_WORKERS = "max-workers"
diff --git a/java_based_implementation/util/exceptions.py
b/paimon_python_java/util/exceptions.py
similarity index 100%
rename from java_based_implementation/util/exceptions.py
rename to paimon_python_java/util/exceptions.py
diff --git a/java_based_implementation/util/java_utils.py
b/paimon_python_java/util/java_utils.py
similarity index 95%
rename from java_based_implementation/util/java_utils.py
rename to paimon_python_java/util/java_utils.py
index 1f0d1ec..6ce6ede 100644
--- a/java_based_implementation/util/java_utils.py
+++ b/paimon_python_java/util/java_utils.py
@@ -16,7 +16,7 @@
# limitations under the License.
################################################################################
-from java_based_implementation.java_gateway import get_gateway
+from paimon_python_java.java_gateway import get_gateway
def to_j_catalog_context(catalog_options: dict):
diff --git a/setup.py b/setup.py
index de23c21..2795cad 100644
--- a/setup.py
+++ b/setup.py
@@ -17,9 +17,10 @@
################################################################################
import fnmatch
-import java_based_implementation.util.setup_utils
+import io
import os
import shutil
+import setup_utils.java_setuputils as java_setuputils
from setuptools import Command, setup
@@ -50,12 +51,12 @@ class CleanCommand(Command):
try:
PACKAGES = [
'paimon_python_api',
- 'java_based_implementation',
- 'java_based_implementation.util'
+ 'paimon_python_java',
+ 'paimon_python_java.util'
]
PACKAGE_DATA = {
- 'java_based_implementation':
java_based_implementation.util.setup_utils.get_package_data()
+ 'paimon_python_java': java_setuputils.get_package_data()
}
install_requires = [
@@ -67,20 +68,25 @@ try:
'pyarrow>=5.0.0'
]
+ with io.open('README.md', 'r', encoding='utf-8') as f:
+ long_description = f.read()
+
setup(
name='paimon_python',
- version='0.9.0.dev1',
+ version='0.1.0.dev0',
packages=PACKAGES,
include_package_data=True,
package_data=PACKAGE_DATA,
cmdclass={'clean': CleanCommand},
install_requires=install_requires,
description='Apache Paimon Python API',
+ long_description=long_description,
+ long_description_content_type='text/markdown',
author='Apache Software Foundation',
author_email='[email protected]',
url='https://paimon.apache.org',
classifiers=[
- 'Development Status :: 5 - Production/Stable',
+ 'Development Status :: 4 - Beta',
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
@@ -89,4 +95,4 @@ try:
python_requires='>=3.8'
)
finally:
- java_based_implementation.util.setup_utils.clean()
+ java_setuputils.clean()
diff --git a/java_based_implementation/util/__init__.py
b/setup_utils/__init__.py
similarity index 93%
rename from java_based_implementation/util/__init__.py
rename to setup_utils/__init__.py
index 65b48d4..27dc0ac 100644
--- a/java_based_implementation/util/__init__.py
+++ b/setup_utils/__init__.py
@@ -15,3 +15,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
+
+"""This module only contains utils for setup and won't be packaged."""
diff --git a/java_based_implementation/util/setup_utils.py
b/setup_utils/java_setuputils.py
old mode 100644
new mode 100755
similarity index 71%
rename from java_based_implementation/util/setup_utils.py
rename to setup_utils/java_setuputils.py
index ec5e367..49e2ea9
--- a/java_based_implementation/util/setup_utils.py
+++ b/setup_utils/java_setuputils.py
@@ -16,23 +16,21 @@
# limitations under the License.
################################################################################
-import importlib
import os
import shutil
import subprocess
-from java_based_implementation.util import constants
-from java_based_implementation.util.constants import PYPAIMON_JAVA_CLASSPATH
+from xml.etree import ElementTree
-_JAVA_IMPL_MODULE = 'java_based_implementation'
+_JAVA_IMPL_MODULE = 'paimon_python_java'
_JAVA_DEPS = 'java_dependencies'
_JAVA_BRIDGE = 'paimon-python-java-bridge'
-# TODO configure version
-_JAVA_BRIDGE_VERSION = '0.9-SNAPSHOT'
+
+_PYPAIMON_TOX_TEST = '_PYPAIMON_TOX_TEST'
def get_package_data():
- is_tox_test = os.environ.get(constants.PYPAIMON_TOX_TEST)
+ is_tox_test = os.environ.get(_PYPAIMON_TOX_TEST)
if is_tox_test and is_tox_test.lower() == "true":
return ['']
@@ -46,18 +44,6 @@ def clean():
shutil.rmtree(java_deps_dir)
-def get_classpath(env):
- user_defined = env.get(PYPAIMON_JAVA_CLASSPATH)
-
- module = importlib.import_module(_JAVA_IMPL_MODULE)
- builtin_java_bridge = os.path.join(*module.__path__, _JAVA_DEPS,
_JAVA_BRIDGE + '.jar')
-
- if user_defined is None:
- return builtin_java_bridge
- else:
- return os.pathsep.join([builtin_java_bridge, user_defined])
-
-
def setup_java_bridge():
java_impl_dir = _find_java_impl_dir()
@@ -79,11 +65,21 @@ def setup_java_bridge():
shutil.copy(
os.path.join(java_bridge_module, 'target/{}-{}.jar'
- .format(_JAVA_BRIDGE, _JAVA_BRIDGE_VERSION)),
+ .format(_JAVA_BRIDGE, _extract_bridge_version())),
java_bridge_dst
)
+def _extract_bridge_version():
+ pom_path = os.path.join(_find_java_impl_dir(), _JAVA_BRIDGE, 'pom.xml')
+ return ElementTree.parse(pom_path).getroot().find(
+ 'POM:version',
+ namespaces={
+ 'POM': 'http://maven.apache.org/POM/4.0.0'
+ }).text
+
+
def _find_java_impl_dir():
- abspath = os.path.abspath(__file__)
- return os.path.dirname(os.path.dirname(abspath))
+ this_dir = os.path.abspath(os.path.dirname(__file__))
+ paimon_python_dir = os.path.dirname(this_dir)
+ return os.path.join(paimon_python_dir, _JAVA_IMPL_MODULE)
diff --git a/tox.ini b/tox.ini
index 562364e..da9b377 100644
--- a/tox.ini
+++ b/tox.ini
@@ -28,6 +28,7 @@ whitelist_externals=
/bin/bash
deps =
-r dev/dev-requirements.txt
+ -e .
passenv = *
commands =
python --version
