This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-4155-ea139cc8277c2aa5764b186f3f8fc50e6a98d830 in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit b1354c8f7c84456f414f2c0dfa464915d3078190 Author: Kevin Liu <[email protected]> AuthorDate: Sun Oct 25 17:54:44 2026 +0000 Bump integration infra to Iceberg 1.12.0 and Spark 4.1.3 (#4155) Also bump hadoop-aws, AWS SDK, Hive metastore client, aws-cli, and pyspark. Hive 4.2 moved IMetaStoreClient into the new hive-standalone-metastore-client artifact, so download it as well. Co-authored-by: Copilot App <[email protected]> --- dev/docker-compose-integration.yml | 2 +- dev/docker-compose.yml | 2 +- dev/spark/Dockerfile | 13 ++++--- pyproject.toml | 4 +- uv.lock | 79 +++++++++++++++++++------------------- 5 files changed, 51 insertions(+), 49 deletions(-) diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 885745644..d217855b4 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -85,7 +85,7 @@ services: retries: 10 start_period: 5s aws-cli: - image: amazon/aws-cli:2.37.2 + image: amazon/aws-cli:2.38.5 container_name: pyiceberg-aws-cli networks: iceberg_net: diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 8583eea10..17c074201 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -33,7 +33,7 @@ services: retries: 10 start_period: 5s aws-cli: - image: amazon/aws-cli:2.37.2 + image: amazon/aws-cli:2.38.5 container_name: pyiceberg-aws-cli depends_on: object-store: diff --git a/dev/spark/Dockerfile b/dev/spark/Dockerfile index 30361b882..eb94c2938 100644 --- a/dev/spark/Dockerfile +++ b/dev/spark/Dockerfile @@ -13,17 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG BASE_IMAGE_SPARK_VERSION=4.0.1 +ARG BASE_IMAGE_SPARK_VERSION=4.1.3 FROM apache/spark:${BASE_IMAGE_SPARK_VERSION} # Dependency versions - keep these compatible # Changing these will invalidate the JAR download cache layer -ARG ICEBERG_VERSION=1.11.0 -ARG ICEBERG_SPARK_RUNTIME_VERSION=4.0_2.13 -ARG HADOOP_VERSION=3.4.1 -ARG AWS_SDK_VERSION=2.24.6 -ARG HIVE_METASTORE_VERSION=4.0.1 +ARG ICEBERG_VERSION=1.12.0 +ARG ICEBERG_SPARK_RUNTIME_VERSION=4.1_2.13 +ARG HADOOP_VERSION=3.4.2 +ARG AWS_SDK_VERSION=2.57.3 +ARG HIVE_METASTORE_VERSION=4.2.1 ARG MAVEN_MIRROR=https://repo.maven.apache.org/maven2 USER root @@ -51,6 +51,7 @@ RUN set -e && \ "org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar" \ "org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" \ "org/apache/hive/hive-standalone-metastore-common/${HIVE_METASTORE_VERSION}/hive-standalone-metastore-common-${HIVE_METASTORE_VERSION}.jar" \ + "org/apache/hive/hive-standalone-metastore-client/${HIVE_METASTORE_VERSION}/hive-standalone-metastore-client-${HIVE_METASTORE_VERSION}.jar" \ "software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar"; \ do \ jar_name=$(basename "${jar_path}") && \ diff --git a/pyproject.toml b/pyproject.toml index f9228ad52..595c6c2ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ dev = [ "moto[server]>=5.0.2", "typing-extensions==4.16.0", "pytest-mock==3.15.1", - "pyspark[connect]==4.0.1", + "pyspark[connect]==4.1.3", "protobuf==7.36.2", # match Spark Connect's gencode "cython>=3.0.0", "deptry>=0.14", @@ -258,7 +258,7 @@ filterwarnings = [ "ignore:subprocess.*is still running:ResourceWarning", # Ignore google-crc32c C extension missing warning (common on Python 3.14+) "ignore:As the c extension couldn't be imported:RuntimeWarning:google_crc32c", - # Ignore Spark 4.0.1 pandas conversion warning under pandas 3.0 + # Ignore Spark pandas conversion warning under pandas 3.0 "ignore:The copy keyword is deprecated and will be removed in a future version.*", ] diff --git a/uv.lock b/uv.lock index 563a8b8c8..21105442a 100644 --- a/uv.lock +++ b/uv.lock @@ -1440,7 +1440,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple"; } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz";, hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -2248,8 +2248,8 @@ name = "httpcore2" version = "2.13.1" source = { registry = "https://pypi.org/simple"; } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "python_full_version < '3.11' or sys_platform != 'emscripten'" }, + { name = "truststore", marker = "python_full_version < '3.11' or sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/f3/1db7aa2bc2524062192bb0e0323969492d1883152a232fe36eea65f4e35c/httpcore2-2.13.1.tar.gz";, hash = "sha256:e0aa977abe17e69a3b820a24542a6fa88702676d83880b8d194dcd18408e5103", size = 68071, upload-time = "2026-09-23T07:47:22.372Z" } wheels = [ @@ -2349,7 +2349,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple"; } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz";, hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -2398,17 +2398,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "exceptiongroup" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions" }, + { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "jedi", marker = "python_full_version < '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, + { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, + { name = "pygments", marker = "python_full_version < '3.11'" }, + { name = "stack-data", marker = "python_full_version < '3.11'" }, + { name = "traitlets", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz";, hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -2434,18 +2434,18 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version >= '3.11'" }, + { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, + { name = "jedi", marker = "python_full_version >= '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, + { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, + { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "stack-data", marker = "python_full_version >= '3.11'" }, + { name = "traitlets", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz";, hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -2457,7 +2457,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple"; } dependencies = [ - { name = "pygments" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz";, hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -4176,10 +4176,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple"; } }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple"; }, marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, + { name = "pytz", marker = "python_full_version < '3.11'" }, + { name = "tzdata", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz";, hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -4251,9 +4251,9 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple"; } }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple"; }, marker = "python_full_version >= '3.11'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, + { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz";, hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -4367,7 +4367,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple"; } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "(python_full_version < '3.11' and sys_platform == 'emscripten') or (python_full_version < '3.11' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz";, hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -5245,7 +5245,7 @@ dev = [ { name = "prek", specifier = ">=0.2.1" }, { name = "protobuf", specifier = "==7.36.2" }, { name = "pyarrow-stubs", specifier = ">=20.0.0.20251107" }, - { name = "pyspark", extras = ["connect"], specifier = "==4.0.1" }, + { name = "pyspark", extras = ["connect"], specifier = "==4.1.3" }, { name = "pytest", specifier = "==9.1.1" }, { name = "pytest-checkdocs", specifier = "==2.14.0" }, { name = "pytest-lazy-fixtures", specifier = "==1.4.1" }, @@ -5414,12 +5414,12 @@ wheels = [ [[package]] name = "pyspark" -version = "4.0.1" +version = "4.1.3" source = { registry = "https://pypi.org/simple"; } dependencies = [ { name = "py4j" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/40/1414582f16c1d7b051c668c2e19c62d21a18bd181d944cb24f5ddbb2423f/pyspark-4.0.1.tar.gz";, hash = "sha256:9d1f22d994f60369228397e3479003ffe2dd736ba79165003246ff7bd48e2c73", size = 434204896, upload-time = "2025-09-06T07:15:57.091Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/ae/33e589ae3cf03c2e9bb78139de00bdfbdc083ebc45b9a4c282f5e33efe70/pyspark-4.1.3.tar.gz";, hash = "sha256:b600238e771e45618a58beaf9d63cfd9aa06f61ce193244cedf890a04a4409f7", size = 455504008, upload-time = "2026-07-15T10:39:01.647Z" } [package.optional-dependencies] connect = [ @@ -5431,6 +5431,7 @@ connect = [ { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple"; }, marker = "python_full_version < '3.11'" }, { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple"; }, marker = "python_full_version >= '3.11'" }, { name = "pyarrow" }, + { name = "zstandard" }, ] [[package]]
