This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-python.git
The following commit(s) were added to refs/heads/main by this push:
new 6581f65 Remove flink-shaded-hadoop-2-uber in java-bridge (#14)
6581f65 is described below
commit 6581f65877145976d4f761b7ea75848f1c24a4a7
Author: yuzelin <[email protected]>
AuthorDate: Wed Oct 9 14:04:55 2024 +0800
Remove flink-shaded-hadoop-2-uber in java-bridge (#14)
---
paimon_python_java/gateway_server.py | 26 ++++++++++++++++++----
.../paimon-python-java-bridge/pom.xml | 8 -------
paimon_python_java/tests/test_data_types.py | 6 ++++-
paimon_python_java/tests/test_write_and_read.py | 11 ++++++---
.../{util/constants.py => tests/utils.py} | 26 +++++++++++++++-------
paimon_python_java/util/constants.py | 1 +
6 files changed, 54 insertions(+), 24 deletions(-)
diff --git a/paimon_python_java/gateway_server.py
b/paimon_python_java/gateway_server.py
index 31933e2..fc05ddc 100644
--- a/paimon_python_java/gateway_server.py
+++ b/paimon_python_java/gateway_server.py
@@ -79,12 +79,30 @@ _JAVA_BRIDGE = 'paimon-python-java-bridge'
def _get_classpath(env):
- user_defined = env.get(constants.PYPAIMON_JAVA_CLASSPATH)
+ classpath = []
module = importlib.import_module(_JAVA_IMPL_MODULE)
builtin_java_bridge = os.path.join(*module.__path__, _JAVA_DEPS,
_JAVA_BRIDGE + '.jar')
+ classpath.append(builtin_java_bridge)
- if user_defined is None:
- return builtin_java_bridge
+ # user defined
+ if constants.PYPAIMON_JAVA_CLASSPATH in env:
+ classpath.append(env[constants.PYPAIMON_JAVA_CLASSPATH])
+
+ # hadoop
+ hadoop_classpath = _get_hadoop_classpath(env)
+ if hadoop_classpath is not None:
+ classpath.append(hadoop_classpath)
+
+ return os.pathsep.join(classpath)
+
+
+def _get_hadoop_classpath(env):
+ if constants.PYPAIMON_HADOOP_CLASSPATH in env:
+ return env[constants.PYPAIMON_HADOOP_CLASSPATH]
+
+ if 'HADOOP_CLASSPATH' in env:
+ return None
else:
- return os.pathsep.join([builtin_java_bridge, user_defined])
+ raise EnvironmentError(f"You haven't set
'{constants.PYPAIMON_HADOOP_CLASSPATH}', \
+ and 'HADOOP_CLASSPATH' is also not set. Ensure one of them is set.")
diff --git a/paimon_python_java/paimon-python-java-bridge/pom.xml
b/paimon_python_java/paimon-python-java-bridge/pom.xml
index 7fe5858..52bc061 100644
--- a/paimon_python_java/paimon-python-java-bridge/pom.xml
+++ b/paimon_python_java/paimon-python-java-bridge/pom.xml
@@ -30,7 +30,6 @@
<properties>
<paimon.version>0.9.0</paimon.version>
- <flink.shaded.hadoop.version>2.8.3-10.0</flink.shaded.hadoop.version>
<py4j.version>0.10.9.7</py4j.version>
<slf4j.version>1.7.32</slf4j.version>
<log4j.version>2.17.1</log4j.version>
@@ -68,12 +67,6 @@
<version>${log4j.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-hadoop-2-uber</artifactId>
- <version>${flink.shaded.hadoop.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
@@ -178,7 +171,6 @@
<include>com.google.flatbuffers:flatbuffers-java</include>
<include>org.slf4j:slf4j-api</include>
<include>org.apache.logging.log4j:log4j-1.2-api</include>
-
<include>org.apache.flink:flink-shaded-hadoop-2-uber</include>
<include>net.sf.py4j:py4j</include>
</includes>
</artifactSet>
diff --git a/paimon_python_java/tests/test_data_types.py
b/paimon_python_java/tests/test_data_types.py
index 72a4587..3920cd2 100644
--- a/paimon_python_java/tests/test_data_types.py
+++ b/paimon_python_java/tests/test_data_types.py
@@ -24,9 +24,9 @@ import tempfile
import pyarrow as pa
import unittest
-
from paimon_python_api import Schema
from paimon_python_java import Catalog
+from paimon_python_java.tests import utils
from paimon_python_java.util import java_utils
from setup_utils import java_setuputils
@@ -36,6 +36,8 @@ class DataTypesTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
java_setuputils.setup_java_bridge()
+ cls.hadoop_path = tempfile.mkdtemp()
+ utils.setup_hadoop_bundle_jar(cls.hadoop_path)
cls.warehouse = tempfile.mkdtemp()
cls.simple_pa_schema = pa.schema([
('f0', pa.int32()),
@@ -47,6 +49,8 @@ class DataTypesTest(unittest.TestCase):
@classmethod
def tearDownClass(cls):
java_setuputils.clean()
+ if os.path.exists(cls.hadoop_path):
+ shutil.rmtree(cls.hadoop_path)
if os.path.exists(cls.warehouse):
shutil.rmtree(cls.warehouse)
diff --git a/paimon_python_java/tests/test_write_and_read.py
b/paimon_python_java/tests/test_write_and_read.py
index cd2bd0e..593fe7e 100644
--- a/paimon_python_java/tests/test_write_and_read.py
+++ b/paimon_python_java/tests/test_write_and_read.py
@@ -22,20 +22,23 @@ import tempfile
import unittest
import pandas as pd
import pyarrow as pa
-import setup_utils.java_setuputils as setuputils
from paimon_python_api import Schema
from paimon_python_java import Catalog
from paimon_python_java.java_gateway import get_gateway
+from paimon_python_java.tests import utils
from paimon_python_java.util import java_utils
from py4j.protocol import Py4JJavaError
+from setup_utils import java_setuputils
class TableWriteReadTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
- setuputils.setup_java_bridge()
+ java_setuputils.setup_java_bridge()
+ cls.hadoop_path = tempfile.mkdtemp()
+ utils.setup_hadoop_bundle_jar(cls.hadoop_path)
cls.warehouse = tempfile.mkdtemp()
cls.simple_pa_schema = pa.schema([
('f0', pa.int32()),
@@ -46,7 +49,9 @@ class TableWriteReadTest(unittest.TestCase):
@classmethod
def tearDownClass(cls):
- setuputils.clean()
+ java_setuputils.clean()
+ if os.path.exists(cls.hadoop_path):
+ shutil.rmtree(cls.hadoop_path)
if os.path.exists(cls.warehouse):
shutil.rmtree(cls.warehouse)
diff --git a/paimon_python_java/util/constants.py
b/paimon_python_java/tests/utils.py
similarity index 61%
copy from paimon_python_java/util/constants.py
copy to paimon_python_java/tests/utils.py
index 00f419c..2af2fe0 100644
--- a/paimon_python_java/util/constants.py
+++ b/paimon_python_java/tests/utils.py
@@ -16,12 +16,22 @@
# limitations under the License.
################################################################################
-# ---------------------------- for env var ----------------------------
-PYPAIMON_CONN_INFO_PATH = '_PYPAIMON_CONN_INFO_PATH'
-PYPAIMON_JVM_ARGS = '_PYPAIMON_JVM_ARGS'
-PYPAIMON_JAVA_CLASSPATH = '_PYPAIMON_JAVA_CLASSPATH'
-PYPAIMON_MAIN_CLASS = 'org.apache.paimon.python.PythonGatewayServer'
-PYPAIMON_MAIN_ARGS = '_PYPAIMON_MAIN_ARGS'
+import os
+import urllib.request
-# ------------------------ for catalog options ------------------------
-MAX_WORKERS = "max-workers"
+from paimon_python_java.util import constants
+
+
+def setup_hadoop_bundle_jar(hadoop_dir):
+ url = 'https://repo.maven.apache.org/maven2/org/apache/flink/' \
+
'flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar'
+
+ response = urllib.request.urlopen(url)
+ if not os.path.exists(hadoop_dir):
+ os.mkdir(hadoop_dir)
+
+ jar_path = os.path.join(hadoop_dir, "bundled-hadoop.jar")
+ with open(jar_path, 'wb') as file:
+ file.write(response.read())
+
+ os.environ[constants.PYPAIMON_HADOOP_CLASSPATH] = jar_path
diff --git a/paimon_python_java/util/constants.py
b/paimon_python_java/util/constants.py
index 00f419c..1039de2 100644
--- a/paimon_python_java/util/constants.py
+++ b/paimon_python_java/util/constants.py
@@ -20,6 +20,7 @@
PYPAIMON_CONN_INFO_PATH = '_PYPAIMON_CONN_INFO_PATH'
PYPAIMON_JVM_ARGS = '_PYPAIMON_JVM_ARGS'
PYPAIMON_JAVA_CLASSPATH = '_PYPAIMON_JAVA_CLASSPATH'
+PYPAIMON_HADOOP_CLASSPATH = '_PYPAIMON_HADOOP_CLASSPATH'
PYPAIMON_MAIN_CLASS = 'org.apache.paimon.python.PythonGatewayServer'
PYPAIMON_MAIN_ARGS = '_PYPAIMON_MAIN_ARGS'
